<commit_before>#!/usr/bin/env python

# Based on js/src/tests/manifest.py

from __future__ import with_statement
import os, re, sys
from subprocess import *

def parse(filename, add_result_callback):
    comment_re = re.compile(r'#.*') # this is wrong, because URLs are allowed to have #, but that works just fine for the fuzzer
    reldir = os.path.dirname(filename)

    with open(filename) as f:
        for line in f:
            sline = comment_re.sub('', line)
            parts = sline.split()
            if len(parts) == 0:
                # line is empty or just a comment, skip
                pass
            elif parts[0] == 'url-prefix':
                pass
            else:
                pos = 0
                while pos < len(parts):
                    K = ['fails', 'skip', 'random', 'asserts', 'fuzzy']
                    part = parts[pos]
                    if any([part.startswith(k) for k in K]):
                        # fails, fails-if(...), asserts(3)
                        #print "Skipping: " + part
                        pos += 1
                        pass
                    elif part == 'include':
                        pos += 1
                        parse(os.path.normpath(os.path.join(reldir, parts[pos])), add_result_callback)
                        break
                    elif part == "needs-focus":
                        pos += 1
                        pass
                    elif part.startswith("require-or("):
                        pos += 1
                        pass
                    elif part.startswith("pref("):
                        pos += 1
                        pass
                    elif part.startswith("HTTP"):
                        pos += 1
                        pass
                    elif part in ["==", "!=", "load"]:
                        pos += 1
                        while pos < len(parts):
                            part = parts[pos]
                            if not part.startswith("data:") and not part.startswith("javascript:") and not part.startswith("about:") and not part.startswith("view-source:"):
                                add_result_callback(os.path.join(reldir, parts[pos]))
                            pos += 1
                        break
                    elif part == "script":
                        break
                    else:
                        print 'warning: in %s unrecognized manifest line element "%s"' % (filename, parts[pos])
                        pos += 1

testfiles = set()

def add_result(r):
    if not ("pngsuite" in r or "351236" in r or "432561" in r or "wrapper.html" in r or "xul" in r or "xbl" in r or r.endswith(".sjs")):
        if r not in testfiles:
            assert r.startswith(sourcetree)
            if not os.path.exists(r.split("?")[0]):
                raise Exception("Missing test: " + r.split("?")[0])
            testfiles.add(r.split("?")[0])

sourcetree = os.path.expanduser("~/trees/mozilla-central/") # XXX assumption alert!

parse(os.path.join(sourcetree, "layout/reftests/reftest.list"), add_result)
parse(os.path.join(sourcetree, "testing/crashtest/crashtests.list"), add_result)

for testfile in testfiles:
    print testfile[len(sourcetree):].lstrip("\\/")
<commit_msg>Update for bug 743105<commit_after>#!/usr/bin/env python

# Based on js/src/tests/manifest.py

from __future__ import with_statement
import os, re, sys
from subprocess import *

def parse(filename, add_result_callback):
    comment_re = re.compile(r'#.*') # this is wrong, because URLs are allowed to have #, but that works just fine for the fuzzer
    reldir = os.path.dirname(filename)

    with open(filename) as f:
        for line in f:
            sline = comment_re.sub('', line)
            parts = sline.split()
            if len(parts) == 0:
                # line is empty or just a comment, skip
                pass
            elif parts[0] == 'url-prefix':
                pass
            else:
                pos = 0
                while pos < len(parts):
                    # http://hg.mozilla.org/mozilla-central/file/866777368827/layout/tools/reftest/print-manifest-dirs.py#l42
                    K = ['fails', 'needs-focus', 'random', 'skip', 'asserts', 'slow', 'require-or', 'silentfail', 'pref', 'test-pref', 'ref-pref', 'fuzzy']
                    part = parts[pos]
                    if any([part.startswith(k) for k in K]):
                        # fails, fails-if(...), asserts(3)
                        #print "Skipping: " + part
                        pos += 1
                        pass
                    elif part == 'include':
                        pos += 1
                        parse(os.path.normpath(os.path.join(reldir, parts[pos])), add_result_callback)
                        break
                    elif part.startswith("HTTP"):
                        pos += 1
                        pass
                    elif part in ["==", "!=", "load"]:
                        pos += 1
                        while pos < len(parts):
                            part = parts[pos]
                            if not part.startswith("data:") and not part.startswith("javascript:") and not part.startswith("about:") and not part.startswith("view-source:"):
                                add_result_callback(os.path.join(reldir, parts[pos]))
                            pos += 1
                        break
                    elif part == "script":
                        break
                    else:
                        print 'warning: in %s unrecognized manifest line element "%s"' % (filename, parts[pos])
                        pos += 1

testfiles = set()

def add_result(r):
    if not ("pngsuite" in r or "351236" in r or "432561" in r or "wrapper.html" in r or "xul" in r or "xbl" in r or r.endswith(".sjs")):
        if r not in testfiles:
            assert r.startswith(sourcetree)
            if not os.path.exists(r.split("?")[0]):
                raise Exception("Missing test: " + r.split("?")[0])
            testfiles.add(r.split("?")[0])

sourcetree = os.path.expanduser("~/trees/mozilla-central/") # XXX assumption alert!

parse(os.path.join(sourcetree, "layout/reftests/reftest.list"), add_result)
parse(os.path.join(sourcetree, "testing/crashtest/crashtests.list"), add_result)

for testfile in testfiles:
    print testfile[len(sourcetree):].lstrip("\\/")
<|endoftext|>


<commit_before>'''
Utilities: parsers, converters, etc.
'''
import re
import logging
from itertools import islice
from module_exceptions import StepperConfigurationError
import math
import gzip

def take(number, iter):
    return list(islice(iter, 0, number))

def parse_duration(duration):
    '''
    Parse duration string, such as '3h2m3s' into milliseconds

    >>> parse_duration('3h2m3s')
    10923000

    >>> parse_duration('0.3s')
    300

    >>> parse_duration('5')
    5000
    '''
    _re_token = re.compile("([0-9.]+)([dhms]?)")

    def parse_token(time, multiplier):
        multipliers = {
            'h': 3600,
            'm': 60,
            's': 1,
        }
        if multiplier:
            if multiplier in multipliers:
                return int(float(time) * multipliers[multiplier] * 1000)
            else:
                raise StepperConfigurationError(
                    'Failed to parse duration: %s' % duration)
        else:
            return int(float(time) * 1000)

    return sum(parse_token(*token) for token in _re_token.findall(duration))

def solve_quadratic(a, b, c):
    '''
    >>> solve_quadratic(1.0, 2.0, 1.0)
    (-1.0, -1.0)
    '''
    discRoot = math.sqrt((b * b) - 4 * a * c)
    root1 = (-b - discRoot) / (2 * a)
    root2 = (-b + discRoot) / (2 * a)
    return (root1, root2)

def s_to_ms(f_sec):
    return int(f_sec * 1000.0)

def get_opener(f_path):
    """ Returns opener function according to file extensions:
        bouth open and gzip.open calls return fileobj.

    Args:
        f_path: str, ammo file path.

    Returns:
        function, to call for file open.
    """
    if f_path.endswith('.gz'):
        logging.info("Using gzip opener")
        return gzip.open
    else:
        return open
<commit_msg>Update util.py<commit_after>'''
Utilities: parsers, converters, etc.
'''
import re
import logging
from itertools import islice
from module_exceptions import StepperConfigurationError
import math
import gzip
import requests
import os
import hashlib

logging.getLogger("requests").setLevel(logging.WARNING)

def take(number, iter):
    return list(islice(iter, 0, number))

def parse_duration(duration):
    '''
    Parse duration string, such as '3h2m3s' into milliseconds

    >>> parse_duration('3h2m3s')
    10923000

    >>> parse_duration('0.3s')
    300

    >>> parse_duration('5')
    5000
    '''
    _re_token = re.compile("([0-9.]+)([dhms]?)")

    def parse_token(time, multiplier):
        multipliers = {
            'h': 3600,
            'm': 60,
            's': 1,
        }
        if multiplier:
            if multiplier in multipliers:
                return int(float(time) * multipliers[multiplier] * 1000)
            else:
                raise StepperConfigurationError(
                    'Failed to parse duration: %s' % duration)
        else:
            return int(float(time) * 1000)

    return sum(parse_token(*token) for token in _re_token.findall(duration))

def solve_quadratic(a, b, c):
    '''
    >>> solve_quadratic(1.0, 2.0, 1.0)
    (-1.0, -1.0)
    '''
    discRoot = math.sqrt((b * b) - 4 * a * c)
    root1 = (-b - discRoot) / (2 * a)
    root2 = (-b + discRoot) / (2 * a)
    return (root1, root2)

def s_to_ms(f_sec):
    return int(f_sec * 1000.0)

def get_opener(f_path):
    """ Returns sub_opener function according to file extensions:
        both open and gzip.open calls return fileobj.

    Args:
        f_path: str, ammo file url.

    Returns:
        object, to call for file open.
    """
    if f_path.startswith("http://") or f_path.startswith("https://"):
        opener = HttpOpener(f_path)
    elif f_path.endswith('.gz'):
        opener = GZOpener(f_path)
    else:
        opener = FSOpener(f_path)
    return opener

class GZOpener(object):

    def __init__(self, f_path):
        self.f_path = f_path

    def __call__(self, *args, **kwargs):
        return gzip.open(*args, **kwargs)

    @property
    def hash(self):
        sep = "|"
        hashed_str = os.path.realpath(self.f_path)
        stat = os.stat(self.f_path)
        cnt = 0
        for stat_option in stat:
            if cnt == 7:  # skip access time
                continue
            cnt += 1
            hashed_str += ";" + str(stat_option)
        hashed_str += ";" + str(os.path.getmtime(self.f_path))
        return hashed_str

    @property
    def data_length(self):
        return os.path.getsize(self.f_path)

class FSOpener(object):

    def __init__(self, f_path):
        self.f_path = f_path

    def __call__(self, *args, **kwargs):
        return open(*args, **kwargs)

    @property
    def hash(self):
        sep = "|"
        hashed_str = os.path.realpath(self.f_path)
        stat = os.stat(self.f_path)
        cnt = 0
        for stat_option in stat:
            if cnt == 7:  # skip access time
                continue
            cnt += 1
            hashed_str += ";" + str(stat_option)
        hashed_str += ";" + str(os.path.getmtime(self.f_path))
        return hashed_str

    @property
    def data_length(self):
        return os.path.getsize(self.f_path)

class HttpOpener(object):
    '''
    downloads small files
    for large files returns wrapped http stream
    '''

    def __init__(self, url):
        self.url = url
        # Meta params
        self.gzip = False
        self.data_info = requests.head(self.url, verify=False)

    def __call__(self, *args, **kwargs):
        return self.open(*args, **kwargs)

    def open(self, *args, **kwargs):
        if self.data_info.status_code == 200:
            self._detect_gzip()
            if not self.gzip \
            and not self.data_length or not str(self.data_length).isdigit() or int(self.data_length) > 10**8):
	            logging.info("Ammofile data is larger than 100MB. Reading from stream..")
                return HttpStreamWrapper(self.url)
            else:
                hasher = hashlib.md5()
                hasher.update(self.hash)
                tmpfile_path = "/tmp/%s" % hasher.hexdigest()
                if os.path.exists(tmpfile_path):
		            logging.info("Ammofile has already been downloaded to %s . Using it..", tmpfile_path)
		        else:
		            logging.info("Downloading ammofile to %s", tmpfile_path)
                    data = requests.get(self.url)
                    f = open(tmpfile_path, "wb")
                    f.write(data.content)
                    f.close()
                if self.gzip:
                    return gzip.open(tmpfile_path, mode='rb')
                else:
                    return open(tmpfile_path, 'rb')
        else:
            raise RuntimeError(
	        "Ammo file not found: %s %s" % (self.data_info.status_code, self.url))

    def _detect_gzip(self):
        stream = requests.get(self.url, stream=True, verify=False)
        if stream.status_code == 200:
            try:
                stream_iterator = stream.raw.stream(2, decode_content=True)
                gz_header = stream_iterator.next()
                if gz_header[:2] == b'\037\213':
		            logging.info("Ammofile data is in gz format")
                    self.gzip = True
            except:
                logging.exception("")
	        finally:
	            stream.connection.close()

    @property
    def hash(self):
        last_modified = self.data_info.headers.get("Last-Modified", '')
        return self.url + "|" + last_modified

    @property
    def data_length(self):
    	try:
    	    data_length = int(self.data_info.headers.get("Content-Length", 0))
    	except:
    	    data_length = 0
        return data_length

class HttpStreamWrapper():
    '''
    makes http stream to look like file object
    '''

    def __init__(self, url):
        self.url = url
        self.buffer = ''
        self.pointer = 0
        self.stream = requests.get(self.url, stream=True, verify=False)
        self._content_consumed = False

    def __enter__(self):
        return self

    def __exit__(self, *args):
        self.stream.connection.close()

    def __iter__(self):
        while True:
            yield self.next()

    def _reopen_stream(self):
        self.stream.connection.close()
        self.stream = requests.get(self.url, stream=True, verify=False)
        self._content_consumed = False

    def _enhance_buffer(self, bytes=10**3):
        self.buffer += self.stream.iter_content(bytes).next()

    def tell(self):
        return self.pointer

    def seek(self, position):
        if self.pointer:
            self.buffer = ''
            self._reopen_stream()
            self._enhance_buffer()
            while len(self.buffer) < position:
                self._enhance_buffer()
            self.pointer = position
            self.buffer = self.buffer[position:]

    def next(self):
        while not '\n' in self.buffer:
            try:
                self._enhance_buffer()
            except (StopIteration, TypeError, requests.exceptions.StreamConsumedError):
                self._content_consumed = True
		        break
        if not self._content_consumed or self.buffer:
    	    try:
                    line = self.buffer[:self.buffer.index('\n')+1]
    	    except ValueError:
        		line = self.buffer
            self.pointer += len(line)
            self.buffer = self.buffer[len(line):]
            return line
        raise StopIteration

    def read(self, chunk_size):
        while len(self.buffer) < chunk_size:
            try:
                self._enhance_buffer()
            except (StopIteration, TypeError, requests.exceptions.StreamConsumedError):
                break
        if len(self.buffer) > chunk_size:
            chunk = self.buffer[:chunk_size]
        else:
            chunk = self.buffer
        self.pointer += len(chunk)
        self.buffer = self.buffer[len(chunk):]
        return chunk

    def readline(self):
        '''
        requests iter_lines() uses splitlines() thus losing '\r\n'
        we need a different behavior for AmmoFileReader
        and we have to use our buffer because we have probably read a bunch into it already
        '''
        try:
            return self.next()
        except StopIteration:
            return ''

<|endoftext|>


<commit_before>#!/usr/bin/env python2

''' Youtube-dlg __main__ file. '''

import sys

if __package__ is None and not hasattr(sys, "frozen"):
    # direct call of __main__.py
    import os.path
    PATH = os.path.realpath(os.path.abspath(__file__))
    sys.path.append(os.path.dirname(os.path.dirname(PATH)))

import youtube_dl_gui

if __name__ == '__main__':
    youtube_dl_gui.main()
<commit_msg>Update docstrings<commit_after>#!/usr/bin/env python2

"""Youtubedlg __main__ file.

__main__ file is a python 'executable' file which calls the youtubedlg
main() function in order to start the app. It can be used for starting
the app from inside the package OR it can be used for starting the app
from a different directory after you have installed the
youtube_dl_gui package.

Example:
    In order to run the app from inside the package.

        $ cd <package director>
        $ python __main__.py

    In order to run the app from /usr/local/bin AFTER
    you have installed the package.

        $ youtube-dlg

"""

import sys

if __package__ is None and not hasattr(sys, "frozen"):
    # direct call of __main__.py
    import os.path
    PATH = os.path.realpath(os.path.abspath(__file__))
    sys.path.append(os.path.dirname(os.path.dirname(PATH)))

import youtube_dl_gui

if __name__ == '__main__':
    youtube_dl_gui.main()
<|endoftext|>


<commit_before>from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame
from math import sin, cos, radians, pi

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

black = Color(0, 1)
thinline = LineStyle(1, black)
white = Color(0xffffff, 1)
gray = Color(0x8c8c8c, 1)
noline = LineStyle(0, black)
cf = RectangleAsset(20, 30, thinline, gray)
laser = RectangleAsset(5, 10, noline, white)
class MC(Sprite):
    """
    Animated space ship
    """

    def __init__(self, position):
        super().__init__(cf, position)
        SpaceGame.listenKeyEvent("keydown", "d", self.dKey)
        SpaceGame.listenKeyEvent("keydown", "a", self.aKey)
        SpaceGame.listenKeyEvent("keydown", "s", self.sKey)
        SpaceGame.listenKeyEvent("keydown", "w", self.wKey)
        SpaceGame.listenKeyEvent("keydown", "e", self.eKey)
        SpaceGame.listenKeyEvent("keydown", "q", self.qKey)
        self.fxcenter = self.fycenter = 0.5
        self.xvector = cos(self.rotation)
        self.yvector = sin(self.rotation)
    def dKey(self, event):
        self.x += cos(self.rotation)
        self.y -= sin(self.rotation)
    def aKey(self, event):
        self.x -= cos(self.rotation)
        self.y += sin(self.rotation)
    def wKey(self, event):
        self.x -= cos((pi/2)-self.rotation)
        self.y -= sin((pi/2)-self.rotation)
    def sKey(self, event):
        self.x += cos((pi/2)-self.rotation)
        self.y += sin((pi/2)-self.rotation)
    def eKey(self, event):
        self.rotation += .09
    def qKey(self, event):
        self.rotation -= .09

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
MC((320,240))
myapp.run()<commit_msg>Updated from Brython Server: 3/27/2017 2:53:20 PM<commit_after>from ggame import App, RectangleAsset, ImageAsset, Sprite, LineStyle, Color, Frame, PolygonAsset
from math import sin, cos, radians, pi

SCREEN_WIDTH = 640
SCREEN_HEIGHT = 480

red = Color(0xff0000, 1.0)
green = Color(0x00ff00, 1.0)
blue = Color(0x0000ff, 1.0)
black = Color(0x000000, 1.0)
white = Color(0xf8f8ff, 1.0)
pwhite = Color(0xfffafa,1.0)
thinline = LineStyle(1, black)
thinline1 = LineStyle(1, pwhite)
noline = LineStyle(0, black)
roof = PolygonAsset([(0,0), (200,0), (100, -120)], thinline, black)
cf = RectangleAsset(20, 30, thinline, gray)
laser = RectangleAsset(5, 10, noline, pwhite)
class MC(Sprite):
    """
    Animated space ship
    """

    def __init__(self, position):
        super().__init__(cf, position)
        SpaceGame.listenKeyEvent("keydown", "d", self.dKey)
        SpaceGame.listenKeyEvent("keydown", "a", self.aKey)
        SpaceGame.listenKeyEvent("keydown", "s", self.sKey)
        SpaceGame.listenKeyEvent("keydown", "w", self.wKey)
        SpaceGame.listenKeyEvent("keydown", "e", self.eKey)
        SpaceGame.listenKeyEvent("keydown", "q", self.qKey)
        self.fxcenter = self.fycenter = 0.5
        self.xvector = cos(self.rotation)
        self.yvector = sin(self.rotation)
    def dKey(self, event):
        self.x += cos(self.rotation)
        self.y -= sin(self.rotation)
    def aKey(self, event):
        self.x -= cos(self.rotation)
        self.y += sin(self.rotation)
    def wKey(self, event):
        self.x -= cos((pi/2)-self.rotation)
        self.y -= sin((pi/2)-self.rotation)
    def sKey(self, event):
        self.x += cos((pi/2)-self.rotation)
        self.y += sin((pi/2)-self.rotation)
    def eKey(self, event):
        self.rotation += .09
    def qKey(self, event):
        self.rotation -= .09

class SpaceGame(App):
    """
    Tutorial4 space game example.
    """
    def __init__(self, width, height):
        super().__init__(width, height)
        black = Color(0, 1)
        noline = LineStyle(0, black)
        bg_asset = RectangleAsset(width, height, noline, black)
        bg = Sprite(bg_asset, (0,0))

myapp = SpaceGame(SCREEN_WIDTH, SCREEN_HEIGHT)
MC((320,240))
myapp.run()<|endoftext|>


<commit_before>#Copyright (C) 2014 OpenBet Limited
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is furnished
#to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from shutit_module import ShutItModule

class docker(ShutItModule):

	def build(self, shutit):
		shutit.install('lsb-release')
		shutit.send('echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -s -c) universe > /etc/apt/sources.list.d/universe.list')
		shutit.send('apt-get update -qq')
		shutit.install('iptables')
		shutit.install('ca-certificates')
		shutit.install('lxc')
		shutit.install('curl')
		shutit.install('aufs-tools')
		shutit.send('pushd /usr/bin')
		shutit.send('curl https://get.docker.io/builds/Linux/x86_64/docker-latest > docker')
		shutit.send('chmod +x docker')
		wrapdocker = """cat > /usr/bin/wrapdocker << 'END'
#!/bin/bash

# First, make sure that cgroups are mounted correctly.
CGROUP=/sys/fs/cgroup

[ -d $CGROUP ] ||
mkdir $CGROUP

mountpoint -q $CGROUP ||
mount -n -t tmpfs -o uid=0, gid=0, mode=0755 cgroup $CGROUP || {
echo "Could not make a tmpfs mount. Did you use -privileged?"
exit 1
}

if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
then
mount -t securityfs none /sys/kernel/security || {
		echo "Could not mount /sys/kernel/security."
		echo "AppArmor detection and -privileged mode might break."
	}
fi

# Mount the cgroup hierarchies exactly as they are in the parent system.
for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
do
		[ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
		mountpoint -q $CGROUP/$SUBSYS ||
				mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS

		# The two following sections address a bug which manifests itself
		# by a cryptic "lxc-start: no ns_cgroup option specified" when
		# trying to start containers withina container.
		# The bug seems to appear when the cgroup hierarchies are not
		# mounted on the exact same directories in the host, and in the
		# container.

		# Named, control-less cgroups are mounted with "-o name=foo"
		# (and appear as such under /proc/<pid>/cgroup) but are usually
		# mounted on a directory named "foo" (without the "name=" prefix).
		# Systemd and OpenRC (and possibly others) both create such a
		# cgroup. To avoid the aforementioned bug, we symlink "foo" to
		# "name=foo". This shouldn't have any adverse effect.
		echo $SUBSYS | grep -q ^name= && {
				NAME=$(echo $SUBSYS | sed s/^name=//)
				ln -s $SUBSYS $CGROUP/$NAME
		}

		# Likewise, on at least one system, it has been reported that
		# systemd would mount the CPU and CPU accounting controllers
		# (respectively "cpu" and "cpuacct") with "-o cpuacct, cpu"
		# but on a directory called "cpu, cpuacct" (note the inversion
		# in the order of the groups). This tries to work around it.
		[ $SUBSYS = cpuacct, cpu ] && ln -s $SUBSYS $CGROUP/cpu, cpuacct
done

# Note: as I write those lines, the LXC userland tools cannot setup
# a "sub-container" properly if the "devices" cgroup is not in its
# own hierarchy. Let's detect this and issue a warning.
grep -q :devices: /proc/1/cgroup ||
echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
grep -qw devices /proc/1/cgroup ||
echo "WARNING: it looks like the 'devices' cgroup is not mounted."

# Now, close extraneous file descriptors.
pushd /proc/self/fd >/dev/null
for FD in *
do
case "$FD" in
# Keep stdin/stdout/stderr
[012])
;;
# Nuke everything else
*)
eval exec "$FD>&-"
;;
esac
done
popd >/dev/null

# If a pidfile is still around (for example after a container restart),
# delete it so that docker can start.
rm -rf /var/run/docker.pid

# If we were given a PORT environment variable, start as a simple daemon;
# otherwise, spawn a shell as well
if [ "$PORT" ]
then
exec docker -d -H 0.0.0.0:$PORT
else

docker -d &
exec bash
fi
END"""
		shutit.send(wrapdocker)
		shutit.send('chmod +x /usr/bin/wrapdocker')
		start_docker = """cat > /root/start_docker.sh << 'END'
#!/bin/bash
/root/start_ssh_server.sh
docker -d &
/usr/bin/wrapdocker
echo "SSH Server up"
echo "Docker daemon running"
END"""
		shutit.send(start_docker)
		shutit.send('chmod +x /root/start_docker.sh')
		shutit.send('popd')
		shutit.send('ln -s /usr/bin/docker /usr/bin/docker.io')
		return True

	def is_installed(self, shutit):
		return False

	def check_ready(self, shutit):
		"""Only apt-based systems are supported support atm.
		"""
		return shutit.cfg['container']['install_type'] == 'apt'

def module():
	return docker(
		'shutit.tk.docker.docker', 0.396,
		description='docker server (communicates with host\'s docker daemon)',
		depends=['shutit.tk.setup', 'shutit.tk.ssh_server.ssh_server']
	)

<commit_msg>clarity<commit_after>#Copyright (C) 2014 OpenBet Limited
#
#Permission is hereby granted, free of charge, to any person obtaining a copy
#of this software and associated documentation files (the "Software"), to deal
#in the Software without restriction, including without limitation the rights
#to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is furnished
#to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

from shutit_module import ShutItModule

class docker(ShutItModule):

	def build(self, shutit):
		shutit.install('lsb-release')
		shutit.send('echo deb http://archive.ubuntu.com/ubuntu $(lsb_release -s -c) universe > /etc/apt/sources.list.d/universe.list')
		shutit.send('apt-get update -qq')
		shutit.install('iptables')
		shutit.install('ca-certificates')
		shutit.install('lxc')
		shutit.install('curl')
		shutit.install('aufs-tools')
		shutit.send('pushd /usr/bin')
		shutit.send('curl https://get.docker.io/builds/Linux/x86_64/docker-latest > docker')
		shutit.send('chmod +x docker')
		wrapdocker = """cat > /usr/bin/wrapdocker << 'END'
#!/bin/bash

# First, make sure that cgroups are mounted correctly.
CGROUP=/sys/fs/cgroup

[ -d $CGROUP ] ||
mkdir $CGROUP

mountpoint -q $CGROUP ||
mount -n -t tmpfs -o uid=0, gid=0, mode=0755 cgroup $CGROUP || {
echo "Could not make a tmpfs mount. Did you use -privileged?"
exit 1
}

if [ -d /sys/kernel/security ] && ! mountpoint -q /sys/kernel/security
then
mount -t securityfs none /sys/kernel/security || {
		echo "Could not mount /sys/kernel/security."
		echo "AppArmor detection and -privileged mode might break."
	}
fi

# Mount the cgroup hierarchies exactly as they are in the parent system.
for SUBSYS in $(cut -d: -f2 /proc/1/cgroup)
do
		[ -d $CGROUP/$SUBSYS ] || mkdir $CGROUP/$SUBSYS
		mountpoint -q $CGROUP/$SUBSYS ||
				mount -n -t cgroup -o $SUBSYS cgroup $CGROUP/$SUBSYS

		# The two following sections address a bug which manifests itself
		# by a cryptic "lxc-start: no ns_cgroup option specified" when
		# trying to start containers withina container.
		# The bug seems to appear when the cgroup hierarchies are not
		# mounted on the exact same directories in the host, and in the
		# container.

		# Named, control-less cgroups are mounted with "-o name=foo"
		# (and appear as such under /proc/<pid>/cgroup) but are usually
		# mounted on a directory named "foo" (without the "name=" prefix).
		# Systemd and OpenRC (and possibly others) both create such a
		# cgroup. To avoid the aforementioned bug, we symlink "foo" to
		# "name=foo". This shouldn't have any adverse effect.
		echo $SUBSYS | grep -q ^name= && {
				NAME=$(echo $SUBSYS | sed s/^name=//)
				ln -s $SUBSYS $CGROUP/$NAME
		}

		# Likewise, on at least one system, it has been reported that
		# systemd would mount the CPU and CPU accounting controllers
		# (respectively "cpu" and "cpuacct") with "-o cpuacct, cpu"
		# but on a directory called "cpu, cpuacct" (note the inversion
		# in the order of the groups). This tries to work around it.
		[ $SUBSYS = cpuacct, cpu ] && ln -s $SUBSYS $CGROUP/cpu, cpuacct
done

# Note: as I write those lines, the LXC userland tools cannot setup
# a "sub-container" properly if the "devices" cgroup is not in its
# own hierarchy. Let's detect this and issue a warning.
grep -q :devices: /proc/1/cgroup ||
echo "WARNING: the 'devices' cgroup should be in its own hierarchy."
grep -qw devices /proc/1/cgroup ||
echo "WARNING: it looks like the 'devices' cgroup is not mounted."

# Now, close extraneous file descriptors.
pushd /proc/self/fd >/dev/null
for FD in *
do
case "$FD" in
# Keep stdin/stdout/stderr
[012])
;;
# Nuke everything else
*)
eval exec "$FD>&-"
;;
esac
done
popd >/dev/null

# If a pidfile is still around (for example after a container restart),
# delete it so that docker can start.
rm -rf /var/run/docker.pid

# If we were given a PORT environment variable, start as a simple daemon;
# otherwise, spawn a shell as well
if [ "$PORT" ]
then
exec docker -d -H 0.0.0.0:$PORT
else

docker -d &
exec bash
fi
END"""
		shutit.send(wrapdocker)
		shutit.send('chmod +x /usr/bin/wrapdocker')
		start_docker = """cat > /root/start_docker.sh << 'END'
#!/bin/bash
/root/start_ssh_server.sh
docker -d &
/usr/bin/wrapdocker
echo "SSH Server up"
echo "Docker daemon running"
END"""
		shutit.send(start_docker)
		shutit.send('chmod +x /root/start_docker.sh')
		shutit.send('popd')
		shutit.send('ln -s /usr/bin/docker /usr/bin/docker.io')
		return True

	def is_installed(self, shutit):
		return False

	def check_ready(self, shutit):
		"""Only apt-based systems are supported support atm.
		"""
		return shutit.cfg['container']['install_type'] == 'apt'

def module():
	return docker(
		'shutit.tk.docker.docker', 0.396,
		description="docker server (communicates with host's docker daemon)",
		depends=['shutit.tk.setup', 'shutit.tk.ssh_server.ssh_server']
	)

<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import StringIO

import logging
import os
import shutil
import zipfile
import urllib2
import urllib
from AmbariConfig import AmbariConfig

logger = logging.getLogger()

class CachingException(Exception):
  pass

class FileCache():
  """
  Provides caching and lookup for service metadata files.
  If service metadata is not available at cache,
  downloads relevant files from the server.
  """

  CLUSTER_CONFIGURATION_CACHE_DIRECTORY="cluster_configuration"
  ALERTS_CACHE_DIRECTORY="alerts"
  RECOVERY_CACHE_DIRECTORY="recovery"
  STACKS_CACHE_DIRECTORY="stacks"
  COMMON_SERVICES_DIRECTORY="common-services"
  CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions"
  HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts"
  HASH_SUM_FILE=".hash"
  ARCHIVE_NAME="archive.zip"
  ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY = "agent.auto.cache.update"

  BLOCK_SIZE=1024*16
  SOCKET_TIMEOUT=10

  def __init__(self, config):
    self.service_component_pool = {}
    self.config = config
    self.cache_dir = config.get('agent', 'cache_dir')
    # Defines whether command should fail when downloading scripts
    # from the server is not possible or agent should rollback to local copy
    self.tolerate_download_failures = \
          config.get('agent','tolerate_download_failures').lower() == 'true'
    self.reset()

  def reset(self):
    self.uptodate_paths = [] # Paths that already have been recently checked

  def get_service_base_dir(self, command, server_url_prefix):
    """
    Returns a base directory for service
    """
    service_subpath = command['commandParams']['service_package_folder']
    return self.provide_directory(self.cache_dir, service_subpath,
                                  server_url_prefix)

  def get_hook_base_dir(self, command, server_url_prefix):
    """
    Returns a base directory for hooks
    """
    try:
      hooks_subpath = command['commandParams']['hooks_folder']
    except KeyError:
      return None
    subpath = os.path.join(self.STACKS_CACHE_DIRECTORY, hooks_subpath)
    return self.provide_directory(self.cache_dir, subpath,
                                  server_url_prefix)

  def get_custom_actions_base_dir(self, server_url_prefix):
    """
    Returns a base directory for custom action scripts
    """
    return self.provide_directory(self.cache_dir,
                                  self.CUSTOM_ACTIONS_CACHE_DIRECTORY,
                                  server_url_prefix)

  def get_custom_resources_subdir(self, command, server_url_prefix):
    """
    Returns a custom directory which must be a subdirectory of the resources dir
    """
    try:
      custom_dir = command['commandParams']['custom_folder']
    except KeyError:
      return None

    return self.provide_directory(self.cache_dir,
                                  custom_dir,
                                  server_url_prefix)

  def get_host_scripts_base_dir(self, server_url_prefix):
    """
    Returns a base directory for host scripts (host alerts, etc) which
    are scripts that are not part of the main agent code
    """
    return self.provide_directory(self.cache_dir,
                                  self.HOST_SCRIPTS_CACHE_DIRECTORY,
                                  server_url_prefix)

  def auto_cache_update_enabled(self):
    if self.config and \
        self.config.has_option(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, FileCache.ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY) and \
            self.config.get(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, FileCache.ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY).lower() == "false":
      return False
    return True

  def provide_directory(self, cache_path, subdirectory, server_url_prefix):
    """
    Ensures that directory at cache is up-to-date. Throws a CachingException
    if any problems occur
    Parameters;
      cache_path: full path to cache directory
      subdirectory: subpath inside cache
      server_url_prefix: url of "resources" folder at the server
    """

    full_path = os.path.join(cache_path, subdirectory)
    logger.debug("Trying to provide directory {0}".format(subdirectory))

    if not self.auto_cache_update_enabled():
      logger.debug("Auto cache update is disabled.")
      return full_path

    try:
      if full_path not in self.uptodate_paths:
        logger.debug("Checking if update is available for "
                     "directory {0}".format(full_path))
        # Need to check for updates at server
        remote_url = self.build_download_url(server_url_prefix,
                                             subdirectory, self.HASH_SUM_FILE)
        memory_buffer = self.fetch_url(remote_url)
        remote_hash = memory_buffer.getvalue().strip()
        local_hash = self.read_hash_sum(full_path)
        if not local_hash or local_hash != remote_hash:
          logger.debug("Updating directory {0}".format(full_path))
          download_url = self.build_download_url(server_url_prefix,
                                                 subdirectory, self.ARCHIVE_NAME)
          membuffer = self.fetch_url(download_url)
          # extract only when the archive is not zero sized
          if (membuffer.getvalue().strip()):
            self.invalidate_directory(full_path)
            self.unpack_archive(membuffer, full_path)
            self.write_hash_sum(full_path, remote_hash)
          else:
            logger.warn("Skipping empty archive: {0}. "
                        "Expected archive was not found. Cached copy will be used.".format(download_url))
            pass
        # Finally consider cache directory up-to-date
        self.uptodate_paths.append(full_path)
    except CachingException, e:
      if self.tolerate_download_failures:
        # ignore
        logger.warn("Error occurred during cache update. "
                    "Error tolerate setting is set to true, so"
                    " ignoring this error and continuing with current cache. "
                    "Error details: {0}".format(str(e)))
      else:
        raise # we are not tolerant to exceptions, command execution will fail
    return full_path

  def build_download_url(self, server_url_prefix,
                         directory, filename):
    """
    Builds up a proper download url for file. Used for downloading files
    from the server.
    directory - relative path
    filename - file inside directory we are trying to fetch
    """
    return "{0}/{1}/{2}".format(server_url_prefix,
                                urllib.pathname2url(directory), filename)

  def fetch_url(self, url):
    """
    Fetches content on url to in-memory buffer and returns the resulting buffer.
    May throw exceptions because of various reasons
    """
    logger.debug("Trying to download {0}".format(url))
    try:
      memory_buffer = StringIO.StringIO()
      proxy_handler = urllib2.ProxyHandler({})
      opener = urllib2.build_opener(proxy_handler)
      u = opener.open(url, timeout=self.SOCKET_TIMEOUT)
      logger.debug("Connected with {0} with code {1}".format(u.geturl(),
                                                             u.getcode()))
      buff = u.read(self.BLOCK_SIZE)
      while buff:
        memory_buffer.write(buff)
        buff = u.read(self.BLOCK_SIZE)
        if not buff:
          break
      return memory_buffer
    except Exception, err:
      raise CachingException("Can not download file from"
                             " url {0} : {1}".format(url, str(err)))

  def read_hash_sum(self, directory):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file) as fh:
        return fh.readline().strip()
    except:
      return None # We don't care

  def write_hash_sum(self, directory, new_hash):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file, "w") as fh:
        fh.write(new_hash)
      os.chmod(hash_file, 0o666)
    except Exception, err:
      raise CachingException("Can not write to file {0} : {1}".format(hash_file,
                                                                 str(err)))

  def invalidate_directory(self, directory):
    """
    Recursively removes directory content (if any). Also, creates
    directory and any parent directories if needed. May throw exceptions
    on permission problems
    """
    logger.debug("Invalidating directory {0}".format(directory))
    try:
      if os.path.exists(directory):
        if os.path.isfile(directory): # It would be a strange situation
          os.unlink(directory)
        elif os.path.isdir(directory):
          shutil.rmtree(directory)
        # create directory itself and any parent directories
      os.makedirs(directory)
    except Exception, err:
      raise CachingException("Can not invalidate cache directory {0}: {1}",
                             directory, str(err))

  def unpack_archive(self, mem_buffer, target_directory):
    """
    Unpacks contents of in-memory buffer to file system.
    In-memory buffer is expected to contain a valid zip archive
    """
    try:
      zfile = zipfile.ZipFile(mem_buffer)
      for name in zfile.namelist():
        (dirname, filename) = os.path.split(name)
        concrete_dir=os.path.abspath(os.path.join(target_directory, dirname))
        if not os.path.isdir(concrete_dir):
          os.makedirs(concrete_dir)
        logger.debug("Unpacking file {0} to {1}".format(name, concrete_dir))
        if filename!='':
          zfile.extract(name, target_directory)
    except Exception, err:
      raise CachingException("Can not unpack zip file to "
                             "directory {0} : {1}".format(
                            target_directory, str(err)))
<commit_msg>AMBARI-19270. Log successful agent cache update at info level. (Attila Doroszlai via stoader)<commit_after>#!/usr/bin/env python

'''
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
import StringIO

import logging
import os
import shutil
import zipfile
import urllib2
import urllib
from AmbariConfig import AmbariConfig

logger = logging.getLogger()

class CachingException(Exception):
  pass

class FileCache():
  """
  Provides caching and lookup for service metadata files.
  If service metadata is not available at cache,
  downloads relevant files from the server.
  """

  CLUSTER_CONFIGURATION_CACHE_DIRECTORY="cluster_configuration"
  ALERTS_CACHE_DIRECTORY="alerts"
  RECOVERY_CACHE_DIRECTORY="recovery"
  STACKS_CACHE_DIRECTORY="stacks"
  COMMON_SERVICES_DIRECTORY="common-services"
  CUSTOM_ACTIONS_CACHE_DIRECTORY="custom_actions"
  HOST_SCRIPTS_CACHE_DIRECTORY="host_scripts"
  HASH_SUM_FILE=".hash"
  ARCHIVE_NAME="archive.zip"
  ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY = "agent.auto.cache.update"

  BLOCK_SIZE=1024*16
  SOCKET_TIMEOUT=10

  def __init__(self, config):
    self.service_component_pool = {}
    self.config = config
    self.cache_dir = config.get('agent', 'cache_dir')
    # Defines whether command should fail when downloading scripts
    # from the server is not possible or agent should rollback to local copy
    self.tolerate_download_failures = \
          config.get('agent','tolerate_download_failures').lower() == 'true'
    self.reset()

  def reset(self):
    self.uptodate_paths = [] # Paths that already have been recently checked

  def get_service_base_dir(self, command, server_url_prefix):
    """
    Returns a base directory for service
    """
    service_subpath = command['commandParams']['service_package_folder']
    return self.provide_directory(self.cache_dir, service_subpath,
                                  server_url_prefix)

  def get_hook_base_dir(self, command, server_url_prefix):
    """
    Returns a base directory for hooks
    """
    try:
      hooks_subpath = command['commandParams']['hooks_folder']
    except KeyError:
      return None
    subpath = os.path.join(self.STACKS_CACHE_DIRECTORY, hooks_subpath)
    return self.provide_directory(self.cache_dir, subpath,
                                  server_url_prefix)

  def get_custom_actions_base_dir(self, server_url_prefix):
    """
    Returns a base directory for custom action scripts
    """
    return self.provide_directory(self.cache_dir,
                                  self.CUSTOM_ACTIONS_CACHE_DIRECTORY,
                                  server_url_prefix)

  def get_custom_resources_subdir(self, command, server_url_prefix):
    """
    Returns a custom directory which must be a subdirectory of the resources dir
    """
    try:
      custom_dir = command['commandParams']['custom_folder']
    except KeyError:
      return None

    return self.provide_directory(self.cache_dir,
                                  custom_dir,
                                  server_url_prefix)

  def get_host_scripts_base_dir(self, server_url_prefix):
    """
    Returns a base directory for host scripts (host alerts, etc) which
    are scripts that are not part of the main agent code
    """
    return self.provide_directory(self.cache_dir,
                                  self.HOST_SCRIPTS_CACHE_DIRECTORY,
                                  server_url_prefix)

  def auto_cache_update_enabled(self):
    if self.config and \
        self.config.has_option(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, FileCache.ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY) and \
            self.config.get(AmbariConfig.AMBARI_PROPERTIES_CATEGORY, FileCache.ENABLE_AUTO_AGENT_CACHE_UPDATE_KEY).lower() == "false":
      return False
    return True

  def provide_directory(self, cache_path, subdirectory, server_url_prefix):
    """
    Ensures that directory at cache is up-to-date. Throws a CachingException
    if any problems occur
    Parameters;
      cache_path: full path to cache directory
      subdirectory: subpath inside cache
      server_url_prefix: url of "resources" folder at the server
    """

    full_path = os.path.join(cache_path, subdirectory)
    logger.debug("Trying to provide directory {0}".format(subdirectory))

    if not self.auto_cache_update_enabled():
      logger.debug("Auto cache update is disabled.")
      return full_path

    try:
      if full_path not in self.uptodate_paths:
        logger.debug("Checking if update is available for "
                     "directory {0}".format(full_path))
        # Need to check for updates at server
        remote_url = self.build_download_url(server_url_prefix,
                                             subdirectory, self.HASH_SUM_FILE)
        memory_buffer = self.fetch_url(remote_url)
        remote_hash = memory_buffer.getvalue().strip()
        local_hash = self.read_hash_sum(full_path)
        if not local_hash or local_hash != remote_hash:
          logger.debug("Updating directory {0}".format(full_path))
          download_url = self.build_download_url(server_url_prefix,
                                                 subdirectory, self.ARCHIVE_NAME)
          membuffer = self.fetch_url(download_url)
          # extract only when the archive is not zero sized
          if (membuffer.getvalue().strip()):
            self.invalidate_directory(full_path)
            self.unpack_archive(membuffer, full_path)
            self.write_hash_sum(full_path, remote_hash)
            logger.info("Updated directory {0}".format(full_path))
          else:
            logger.warn("Skipping empty archive: {0}. "
                        "Expected archive was not found. Cached copy will be used.".format(download_url))
            pass
        # Finally consider cache directory up-to-date
        self.uptodate_paths.append(full_path)
    except CachingException, e:
      if self.tolerate_download_failures:
        # ignore
        logger.warn("Error occurred during cache update. "
                    "Error tolerate setting is set to true, so"
                    " ignoring this error and continuing with current cache. "
                    "Error details: {0}".format(str(e)))
      else:
        raise # we are not tolerant to exceptions, command execution will fail
    return full_path

  def build_download_url(self, server_url_prefix,
                         directory, filename):
    """
    Builds up a proper download url for file. Used for downloading files
    from the server.
    directory - relative path
    filename - file inside directory we are trying to fetch
    """
    return "{0}/{1}/{2}".format(server_url_prefix,
                                urllib.pathname2url(directory), filename)

  def fetch_url(self, url):
    """
    Fetches content on url to in-memory buffer and returns the resulting buffer.
    May throw exceptions because of various reasons
    """
    logger.debug("Trying to download {0}".format(url))
    try:
      memory_buffer = StringIO.StringIO()
      proxy_handler = urllib2.ProxyHandler({})
      opener = urllib2.build_opener(proxy_handler)
      u = opener.open(url, timeout=self.SOCKET_TIMEOUT)
      logger.debug("Connected with {0} with code {1}".format(u.geturl(),
                                                             u.getcode()))
      buff = u.read(self.BLOCK_SIZE)
      while buff:
        memory_buffer.write(buff)
        buff = u.read(self.BLOCK_SIZE)
        if not buff:
          break
      return memory_buffer
    except Exception, err:
      raise CachingException("Can not download file from"
                             " url {0} : {1}".format(url, str(err)))

  def read_hash_sum(self, directory):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file) as fh:
        return fh.readline().strip()
    except:
      return None # We don't care

  def write_hash_sum(self, directory, new_hash):
    """
    Tries to read a hash sum from previously generated file. Returns string
    containing hash or None
    """
    hash_file = os.path.join(directory, self.HASH_SUM_FILE)
    try:
      with open(hash_file, "w") as fh:
        fh.write(new_hash)
      os.chmod(hash_file, 0o666)
    except Exception, err:
      raise CachingException("Can not write to file {0} : {1}".format(hash_file,
                                                                 str(err)))

  def invalidate_directory(self, directory):
    """
    Recursively removes directory content (if any). Also, creates
    directory and any parent directories if needed. May throw exceptions
    on permission problems
    """
    logger.debug("Invalidating directory {0}".format(directory))
    try:
      if os.path.exists(directory):
        if os.path.isfile(directory): # It would be a strange situation
          os.unlink(directory)
        elif os.path.isdir(directory):
          shutil.rmtree(directory)
        # create directory itself and any parent directories
      os.makedirs(directory)
    except Exception, err:
      raise CachingException("Can not invalidate cache directory {0}: {1}",
                             directory, str(err))

  def unpack_archive(self, mem_buffer, target_directory):
    """
    Unpacks contents of in-memory buffer to file system.
    In-memory buffer is expected to contain a valid zip archive
    """
    try:
      zfile = zipfile.ZipFile(mem_buffer)
      for name in zfile.namelist():
        (dirname, filename) = os.path.split(name)
        concrete_dir=os.path.abspath(os.path.join(target_directory, dirname))
        if not os.path.isdir(concrete_dir):
          os.makedirs(concrete_dir)
        logger.debug("Unpacking file {0} to {1}".format(name, concrete_dir))
        if filename!='':
          zfile.extract(name, target_directory)
    except Exception, err:
      raise CachingException("Can not unpack zip file to "
                             "directory {0} : {1}".format(
                            target_directory, str(err)))
<|endoftext|>


<commit_before># Generated by Django 2.2.18 on 2021-05-16 19:21

import django.contrib.postgres.fields
from django.db import migrations, models
from django.db.models import F, Sum

def move_extensions(apps, schema_editor):
    """Move existing `extended` value into the new field `extensions`."""
    Membership = apps.get_model('workshops', 'Membership')
    for membership in Membership.objects.all():
        membership.extensions = (
            [] if membership.extended is None
            else [membership.extended]
        )
        membership.save()

def undo_extensions(apps, schema_editor):
    """Move from `extensions` to single value `extended` field."""
    Membership = apps.get_model('workshops', 'Membership')
    for membership in Membership.objects.all():
        membership.extended = sum(membership.extensions) or None
        membership.save()

class Migration(migrations.Migration):

    dependencies = [
        ('workshops', '0243_auto_20210508_1954'),
    ]

    operations = [
        migrations.AddField(
            model_name='membership',
            name='extensions',
            field=django.contrib.postgres.fields.ArrayField(
                base_field=models.PositiveIntegerField(),
                default=list,
                help_text='Number of days the agreement was extended with. The field stores multiple extensions. The agreement end date has been moved by a cumulative number of days from this field.',
                size=None,
            ),
            preserve_default=False,
        ),
        migrations.RunPython(move_extensions, undo_extensions),
        migrations.RemoveField(model_name='membership', name='extended'),
    ]
<commit_msg>[#1937] Fix Membership.extensions migration<commit_after># Generated by Django 2.2.18 on 2021-05-17 06:36

import django.contrib.postgres.fields
from django.db import migrations, models

def move_extensions(apps, schema_editor):
    """Move existing `extended` value into the new field `extensions`."""
    Membership = apps.get_model('workshops', 'Membership')
    for membership in Membership.objects.all():
        membership.extensions = (
            [] if membership.extended is None
            else [membership.extended]
        )
        membership.save()

def undo_extensions(apps, schema_editor):
    """Move from `extensions` to single value `extended` field."""
    Membership = apps.get_model('workshops', 'Membership')
    for membership in Membership.objects.all():
        membership.extended = sum(membership.extensions) or None
        membership.save()

class Migration(migrations.Migration):

    dependencies = [
        ('workshops', '0243_auto_20210508_1954'),
    ]

    operations = [
        migrations.AddField(
            model_name='membership',
            name='extensions',
            field=django.contrib.postgres.fields.ArrayField(base_field=models.PositiveIntegerField(), default=list, help_text='Number of days the agreement was extended with. The field stores multiple extensions. The agreement end date has been moved by a cumulative number of days from this field.', size=None),
        ),
        migrations.RunPython(move_extensions, undo_extensions),
        migrations.RemoveField(
            model_name='membership',
            name='extended',
        ),
    ]
<|endoftext|>


<commit_before>from functools import wraps
from flask import request,jsonify
from nishu import get_application_model
import  hmac, hashlib

def login_required(func):

    @wraps(func)
    def decorator_func(*args,**kwargs):
        user        = request.headers.get('user')
        api_key     = request.headers.get('api_key')
        api_secret  = request.headers.get('api_secret')
        user_hash   = request.headers.get('hash')
        user_timestamp   = request.headers.get('timestamp')

        if not user or not api_key or not api_secret :
            return jsonify("Error: Invalid Request"),412

        if not hash or not user_timestamp or not user_hash:
            return jsonify("Error: Invalid Request"), 412

        server_key = get_key(api_key)

        if not server_key:
            return jsonify("key not found"),412

        #for get request

        if request.method == 'GET':
            url = request.path + '?' + request.query_string if request.query_string else request.path
            timestamp_hash = generate_hmac(str(server_key), str(user_timestamp))
            server_hash = generate_hmac(str(timestamp_hash), str(url))
            if hmac.compare_digest(server_hash, user_hash):
                return func(*args, **kwargs)
            else:
                return jsonify("Error : HMAC is not matched"), 412
            #change with the hmac
            # server_hash = base64.base64encode(str(server_key),str(url))
            # if user_hash == server_hash:
            #     return func(*args,**kwargs)

            # else :
                # return jsonify("Error: HMAC is not matched"),412

        if request.method == 'POST':
            return ""
    return decorator_func

def get_key(api_key):
    app_model= get_application_model(api_key)
    #need validation
    server_key = app_model.api_key
    return server_key

def generate_hmac(key,message):
    hash = hmac.new(key,message.encode('utf-8'),hashlib.sha256).hexdigest()
    return hash
<commit_msg>added authentication for post protocol<commit_after>from functools import wraps
from flask import request,jsonify
from nishu import get_application_model
import  hmac, hashlib

def login_required(func):

    @wraps(func)
    def decorator_func(*args,**kwargs):
        user        = request.headers.get('user')
        api_key     = request.headers.get('api_key')
        api_secret  = request.headers.get('api_secret')
        user_hash   = request.headers.get('hash')
        user_timestamp   = request.headers.get('timestamp')

        if not user or not api_key or not api_secret :
            return jsonify("Error: Invalid Request"),412

        if not hash or not user_timestamp or not user_hash:
            return jsonify("Error: Invalid Request"), 412

        server_key = get_key(api_key)

        if not server_key:
            return jsonify("key not found"),412

        timestamp_hash = generate_hmac(str(server_key), str(user_timestamp))
        #for get request

        if request.method == 'GET':
            url = request.path + '?' + request.query_string if request.query_string else request.path
            server_hash = generate_hmac(str(timestamp_hash), str(url))
            if hmac.compare_digest(server_hash, user_hash):
                return func(*args, **kwargs)
            else:
                return jsonify("Error : HMAC is not matched"), 412
            #change with the hmac
            # server_hash = base64.base64encode(str(server_key),str(url))
            # if user_hash == server_hash:
            #     return func(*args,**kwargs)

            # else :
                # return jsonify("Error: HMAC is not matched"),412

        if request.method == 'POST':
            #check for file upload
            data = request.data.decode('utf-8')
            server_hash = generate_hmac(str(timestamp_hash),data)
            if hmac.compare_digest(server_hash,user_hash):
                return func(*args, **kwargs)
            else:
                return jsonify("Error : HMAC is not matched"), 412
    return decorator_func

def get_key(api_key):
    app_model= get_application_model(api_key)
    #need validation
    server_key = app_model.api_key
    return server_key

def generate_hmac(key,message):
    hash = hmac.new(key,message.encode('utf-8'),hashlib.sha256).hexdigest()
    return hash
<|endoftext|>


<commit_before>#! /usr/bin/env python

# This file is part of the dune-pymor project:
#   https://github.com/pyMor/dune-pymor
# Copyright Holders: Felix Albrecht, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

import pybindgen

class Exceptions():

    def __init__(self,
                 PymorException,
                 types_are_not_compatible,
                 sizes_do_not_match,
                 index_out_of_range,
                 key_is_not_valid):
        self.PymorException = PymorException
        self.types_are_not_compatible = types_are_not_compatible
        self.sizes_do_not_match = sizes_do_not_match
        self.index_out_of_range = index_out_of_range
        self.key_is_not_valid = key_is_not_valid

def add_Exceptions(module):
    assert(isinstance(module, pybindgen.module.Module))
    namespace = module.add_cpp_namespace('Dune')
    PymorException = namespace.add_exception('Dune::PymorException')
    namespace = namespace.add_cpp_namespace('Pymor').add_cpp_namespace('Exception')
    types_are_not_compatible = namespace.add_exception('types_are_not_compatible',
                                                       parent=PymorException,
                                                       message_rvalue='%(EXC)s.pymorWhat()')
    sizes_do_not_match = namespace.add_exception('sizes_do_not_match',
                                                 parent=PymorException,
                                                 message_rvalue='%(EXC)s.pymorWhat()')
    index_out_of_range = namespace.add_exception('index_out_of_range',
                                                 parent=PymorException,
                                                 message_rvalue='%(EXC)s.pymorWhat()')
    key_is_not_valid = namespace.add_exception('key_is_not_valid',
                                               parent=PymorException,
                                               message_rvalue='%(EXC)s.pymorWhat()')
    return module, Exceptions(PymorException,
                              types_are_not_compatible,
                              sizes_do_not_match,
                              index_out_of_range,
                              key_is_not_valid)
<commit_msg>[dune.pymor.common.exceptions]<commit_after>#! /usr/bin/env python

# This file is part of the dune-pymor project:
#   https://github.com/pyMor/dune-pymor
# Copyright Holders: Felix Albrecht, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

import pybindgen

class Exceptions(object):

    def __init__(self,
                 PymorException,
                 types_are_not_compatible,
                 sizes_do_not_match,
                 index_out_of_range,
                 key_is_not_valid):
        self.PymorException = PymorException
        self.types_are_not_compatible = types_are_not_compatible
        self.sizes_do_not_match = sizes_do_not_match
        self.index_out_of_range = index_out_of_range
        self.key_is_not_valid = key_is_not_valid

def add_Exceptions(module):
    assert(isinstance(module, pybindgen.module.Module))
    namespace = module.add_cpp_namespace('Dune')
    PymorException = namespace.add_exception('Dune::PymorException')
    namespace = namespace.add_cpp_namespace('Pymor').add_cpp_namespace('Exception')
    types_are_not_compatible = namespace.add_exception('types_are_not_compatible',
                                                       parent=PymorException,
                                                       message_rvalue='%(EXC)s.pymorWhat()')
    sizes_do_not_match = namespace.add_exception('sizes_do_not_match',
                                                 parent=PymorException,
                                                 message_rvalue='%(EXC)s.pymorWhat()')
    index_out_of_range = namespace.add_exception('index_out_of_range',
                                                 parent=PymorException,
                                                 message_rvalue='%(EXC)s.pymorWhat()')
    key_is_not_valid = namespace.add_exception('key_is_not_valid',
                                               parent=PymorException,
                                               message_rvalue='%(EXC)s.pymorWhat()')
    return module, Exceptions(PymorException,
                              types_are_not_compatible,
                              sizes_do_not_match,
                              index_out_of_range,
                              key_is_not_valid)
<|endoftext|>


<commit_before>from ..config import get_config, save_config

def list_config():
     yield str(get_config())

def set_value(key, value):
    config = get_config()
    config[key] = value
    save_config(config)
    yield 'set {} to {} in your config'.format(key, value)

<commit_msg>add in safe guards to stop people from adding new keys and changing non string keys<commit_after>from ..config import get_config, save_config

def list_config():
     yield str(get_config())

def set_value(key, value):
    config = get_config()
    if key not in config:
        yield 'key {} is not in config. Keep your config clean'.format(key)
    elif not isinstance(config[key], basestring):
        yield 'You can only modify string values in your config. {} has type {}'.format(key, type(config[key]))
    else:
        config[key] = value
        save_config(config)
        yield 'set {} to {} in your config'.format(key, value)

<|endoftext|>


<commit_before>import os
import logging
from subprocess import check_call, CalledProcessError

from ... import constants
from ...config import get_config_value
from ...subprocess import check_call_demoted, check_and_log_output_and_error_demoted
from ...source import Repo
from ...path import parent_dir
from ...log import log_to_client
from dusty.compiler.spec_assembler import get_repo_of_app_or_library, get_assembled_specs

def _ensure_vm_dir_exists(remote_dir):
    check_call_demoted(['boot2docker', 'ssh', 'sudo mkdir -p {0}; sudo chown -R docker {0}'.format(remote_dir)])

def _rsync_command(local_path, remote_path, is_dir=True, from_local=True, exclude_git=True):
    ssh_opts = 'ssh -p 2022 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /Users/{}/.ssh/id_boot2docker'.format(get_config_value(constants.CONFIG_MAC_USERNAME_KEY))
    command = ['rsync', '-e', ssh_opts, '-az', '--force']
    if exclude_git:
        command += ['--exclude', '*/.git']
    if from_local:
        path_args = ['{}{}'.format(local_path, '/' if is_dir else ''), 'docker@localhost:{}'.format(remote_path)]
    else:
        path_args = ['docker@localhost:{}{}'.format(remote_path, '/' if is_dir else ''), local_path]
    command += path_args
    return command

def vm_path_is_directory(remote_path):
    """A weak check of whether a path in the boot2docker VM is a directory.
    This function returns False on any process error, so False may indicate
    other failures such as the path not actually existing."""
    try:
        check_call_demoted(['boot2docker', 'ssh', 'test -d {}'.format(remote_path)])
    except CalledProcessError:
        return False
    return True

def sync_local_path_to_vm(local_path, remote_path, demote=False):
    is_dir = os.path.isdir(local_path)
    _ensure_vm_dir_exists(remote_path if is_dir else parent_dir(remote_path))
    command = _rsync_command(local_path, remote_path, is_dir=is_dir)
    logging.debug('Executing rsync command: {}'.format(' '.join(command)))
    check_call(command) if not demote else check_and_log_output_and_error_demoted(command)

def sync_local_path_from_vm(local_path, remote_path, demote=False, is_dir=True):
    command = _rsync_command(local_path, remote_path, is_dir=is_dir, from_local=False)
    logging.debug('Executing rsync command: {}'.format(' '.join(command)))
    check_call(command) if not demote else check_and_log_output_and_error_demoted(command)

def sync_repos(repos):
    logging.info('Syncing repos over rsync')
    for repo in repos:
        repo_type = 'overridden' if repo.is_overridden else 'Dusty-managed'
        log_to_client('Syncing {} repo {} to remote at {}'.format(repo_type, repo.remote_path, repo.vm_path))
        sync_local_path_to_vm(repo.local_path, repo.vm_path)

# For the bottom two functions to work, they really need to take already expanded specs
def sync_repos_by_app_name(expanded_specs, app_names):
    repos = set()
    for app_name in app_names:
        for lib_name in expanded_specs['apps'][app_name].get('depends', {}).get('libs', []):
            repos.add(get_repo_of_app_or_library(lib_name))
        repos.add(get_repo_of_app_or_library(app_name))
    sync_repos(repos)

def sync_repos_by_lib_name(expanded_specs, lib_names):
    repos = set()
    for lib_name in lib_names:
        for depends_lib_name in expanded_specs['libs'][lib_name].get('depends', {}).get('libs', []):
            repos.add(get_repo_of_app_or_library(depends_lib_name))
        repos.add(get_repo_of_app_or_library(lib_name))
    sync_repos(repos)
<commit_msg>reduce copied code<commit_after>import os
import logging
from subprocess import check_call, CalledProcessError

from ... import constants
from ...config import get_config_value
from ...subprocess import check_call_demoted, check_and_log_output_and_error_demoted
from ...source import Repo
from ...path import parent_dir
from ...log import log_to_client
from dusty.compiler.spec_assembler import get_repo_of_app_or_library, get_assembled_specs

def _ensure_vm_dir_exists(remote_dir):
    check_call_demoted(['boot2docker', 'ssh', 'sudo mkdir -p {0}; sudo chown -R docker {0}'.format(remote_dir)])

def _rsync_command(local_path, remote_path, is_dir=True, from_local=True, exclude_git=True):
    ssh_opts = 'ssh -p 2022 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /Users/{}/.ssh/id_boot2docker'.format(get_config_value(constants.CONFIG_MAC_USERNAME_KEY))
    command = ['rsync', '-e', ssh_opts, '-az', '--force']
    if exclude_git:
        command += ['--exclude', '*/.git']
    if from_local:
        path_args = ['{}{}'.format(local_path, '/' if is_dir else ''), 'docker@localhost:{}'.format(remote_path)]
    else:
        path_args = ['docker@localhost:{}{}'.format(remote_path, '/' if is_dir else ''), local_path]
    command += path_args
    return command

def vm_path_is_directory(remote_path):
    """A weak check of whether a path in the boot2docker VM is a directory.
    This function returns False on any process error, so False may indicate
    other failures such as the path not actually existing."""
    try:
        check_call_demoted(['boot2docker', 'ssh', 'test -d {}'.format(remote_path)])
    except CalledProcessError:
        return False
    return True

def sync_local_path_to_vm(local_path, remote_path, demote=False):
    is_dir = os.path.isdir(local_path)
    _ensure_vm_dir_exists(remote_path if is_dir else parent_dir(remote_path))
    command = _rsync_command(local_path, remote_path, is_dir=is_dir)
    logging.debug('Executing rsync command: {}'.format(' '.join(command)))
    check_call(command) if not demote else check_and_log_output_and_error_demoted(command)

def sync_local_path_from_vm(local_path, remote_path, demote=False, is_dir=True):
    command = _rsync_command(local_path, remote_path, is_dir=is_dir, from_local=False)
    logging.debug('Executing rsync command: {}'.format(' '.join(command)))
    check_call(command) if not demote else check_and_log_output_and_error_demoted(command)

def sync_repos(repos):
    logging.info('Syncing repos over rsync')
    for repo in repos:
        repo_type = 'overridden' if repo.is_overridden else 'Dusty-managed'
        log_to_client('Syncing {} repo {} to remote at {}'.format(repo_type, repo.remote_path, repo.vm_path))
        sync_local_path_to_vm(repo.local_path, repo.vm_path)

# For the sync_repos_by_... functions to work, they really need to take already expanded specs
def _sync_repos_by_type_name(expanded_specs, type_names, dusty_type):
    repos = set()
    for type_name in type_names:
        for lib_name in expanded_specs[dusty_type][type_name].get('depends', {}).get('libs', []):
            repos.add(get_repo_of_app_or_library(lib_name))
        repos.add(get_repo_of_app_or_library(type_name))
    sync_repos(repos)

def sync_repos_by_app_name(expanded_specs, app_names):
    _sync_repos_by_type_name(expanded_specs, app_names, 'apps')

def sync_repos_by_lib_name(expanded_specs, lib_names):
    _sync_repos_by_type_name(expanded_specs, lib_names, 'libs')

<|endoftext|>


<commit_before>from bs4 import BeautifulSoup
import os

linkCount = 0
parents = []
hrefs = []

controlList = ["b", "small", "span", "h1,", "h2", "h3", "h4", "h5", "i"]

rawFile = "/home/ubuntu/warc-test/part-00000"

inputData = open(rawFile, "rb")
inputText = inputData.read()

startFix = "" + inputText.split("", 1)[1]
endFix = startFix.rsplit("</html", 1)[0] + ""

pageCount = 0
for thing in endFix.split(""):
	page = "" + thing
	soup = BeautifulSoup(page)
	for link in soup("a"):
		linkCount += 1
		href = link.get('href', None)
		hrefs.append(href)
		parentTag = link.parent.name

		parentMatch = False
		for item in parents:
			if  parentTag in item:
				parentMatch = True
				item[1] = item[1] + 1
		if parentMatch == False:
			parents.append([parentTag, 1])

	print "Total Links: " + str(linkCount)
	pageCount += 1
	print pageCount
	print parents

inputData.close()

<commit_msg>fixed typo<commit_after>from bs4 import BeautifulSoup
import os

linkCount = 0
parents = []
hrefs = []

controlList = ["b", "small", "span", "h1,", "h2", "h3", "h4", "h5", "i"]

rawFile = "/home/ubuntu/warc-test/part-00000"

inputData = open(rawFile, "rb")
inputText = inputData.read()

startFix = "" + inputText.split("", 1)[1]
endFix = startFix.rsplit("", 1)[0] + ""

pageCount = 0
for thing in endFix.split(""):
	page = "" + thing
	soup = BeautifulSoup(page)
	for link in soup("a"):
		linkCount += 1
		href = link.get('href', None)
		hrefs.append(href)
		parentTag = link.parent.name

		parentMatch = False
		for item in parents:
			if  parentTag in item:
				parentMatch = True
				item[1] = item[1] + 1
		if parentMatch == False:
			parents.append([parentTag, 1])

	print "Total Links: " + str(linkCount)
	pageCount += 1
	print pageCount
	print parents

inputData.close()

<|endoftext|>


<commit_before>#! /usr/bin/python3

import sys

PAD_LIST = ["(", ")", ",", ".", ":", "{", "}", "[", "]", "=", "+", "-", ";", "//", "/*", "*/"]

def find(line, text):
	if line.find(text) != -1:
		return True

def load_file(path, filename, operation="r"):
	file = []
	with open(path + filename, operation) as open_file:
		file = open_file.readlines()
	return file

def parse_file(loaded_file, pad_list):
	parsed = []

	for line in loaded_file:
		clean = line.strip().strip("\n")
		cleaned = []

		for item in pad_list:
			clean = clean.replace(item, " " + item + " ") # Should I use str.format here?

		clean = clean.split(" ")
		# Keep track if we pass the single line comment syntax
		isComment = False
		for item in clean:
			if item is not "":
				# Special for single line comments
				if isComment:
					item = "//" + item
				if item.strip() == "//":
					isComment = True
				cleaned.append(item)

		if cleaned != []:
			parsed.extend(cleaned)

	return parsed

# parsed_file should be reversed at least when you call it from outside the function
# parent_objects should be a dict
def format_parsed_file(parsed_file):
	tree = {}

	bracket_pairs = []
	pair = []

	for index in range(len(parsed_file)):
		word = parsed_file[index]

		if word == "{":
			pair.append(index)
		elif word == "}":
			pair.append(index)

		if len(pair) == 2:
			bracket_pairs.append(pair)
			pair = []

	print(bracket_pairs)

	return tree

#print(format_parsed_file(parse_file(load_file("", "test_file.txt"), PAD_LIST).reverse()))
parsed = parse_file(load_file("", "test_file.txt"), PAD_LIST)
print(parsed)

formated = format_parsed_file(parsed)
print(formated)
<commit_msg>Good progress on doc.py<commit_after>#! /usr/bin/python3

import sys

PAD_LIST = ["(", ")", ",", ".", ":", "{", "}", "[", "]", "=", "+", "-", ";", "//", "/*", "*/"]

def find(line, text):
	if line.find(text) != -1:
		return True

def load_file(path, filename, operation="r"):
	file = []
	with open(path + filename, operation) as open_file:
		file = open_file.readlines()
	return file

def parse_file(loaded_file, pad_list):
	parsed = []

	for line in loaded_file:
		clean = line.strip().strip("\n")
		cleaned = []

		for item in pad_list:
			clean = clean.replace(item, " " + item + " ") # Should I use str.format here?

		clean = clean.split(" ")
		# Keep track if we pass the single line comment syntax
		isComment = False
		for item in clean:
			if item is not "":
				if item.strip() == "//":
					isComment = True
				if not isComment:
					cleaned.append(item)

		if cleaned != []:
			parsed.extend(cleaned)

	return parsed

def make_thought(lines):
	pass

def parse_new(loaded_file):
	#print(loaded_file)
	thought = {
		"head": "",
		"parameters": [],
		"body": {}
	}

	cleaned = []
	for line in loaded_file:
		striped = line.strip().split("//")[0]
		if striped != "":
			cleaned.append(striped)

	level = 0
	group = {}
	depth = []
	ignore = False
	has_local = False
	get_local_defaults = False
	level_map = []

	for line in cleaned:
		if line.find("{") != -1:
			level += 1
			level_map.append(level)
		if line.find("}") != -1:
			level -= 1
			level_map.append(level)

		if line.count("function ") and not ignore:
			marker = group

			for foot in depth:
				marker = marker[foot]
			marker[line] = {}
			depth.append(line)
			print(line)

		if line.count("localContainer = "):
			marker = group
			for foot in depth:
				marker = marker[foot]
			print(line)

		if line.count("localContainer.") and line.count("= function") and not line.count("var"):
			marker = group
			for foot in depth:
				marker = marker[foot]
			t = line.split(" ")[0]
			print("->", t)
			marker[t] = []

		if get_local_defaults:
			marker = group
			for foot in depth:
				marker = marker[foot]
			#print(marker)
			marker.append(line)
			print("- - ->", line)

		if line.count("var local ="):
			get_local_defaults = True
			marker = group
			for foot in depth:
				marker = marker[foot]
			marker["defaults"] = []
			depth.append(t)

		if get_local_defaults and line.count("};"):
			get_local_defaults = False

		if line.count("local.") and line.count("= function"):
			marker = group
			for foot in depth:
				marker = marker[foot]
			marker.append(line)
			print("- ->", line)
			ignore = True

		if line.count("return local;"):
			ignore = False
			depth.pop()

		if line.count("return localContainer") and not line.count("="):
			depth.pop()
			print(line)

	print(group)
	print(level_map)

# parsed_file should be reversed at least when you call it from outside the function
# parent_objects should be a dict
def format_parsed_file(parsed_file):

	for word in parsed_file:
		print(word)

#print(format_parsed_file(parse_file(load_file("", "test_file.txt"), PAD_LIST).reverse()))
loaded = load_file("", "test_file.txt")
parsed = parse_file(loaded, PAD_LIST)
#print(parsed)

#formated = format_parsed_file(parsed)
#print(formated)

parse_new(loaded)
<|endoftext|>