<commit_before>import sublime
import sublime_plugin
from os import path
import re
from collections import namedtuple

AssertionLineDetails = namedtuple(
    'AssertionLineDetails', ['comment_marker_match', 'assertion_colrange', 'line_region']
)

def get_syntax_test_tokens(view):
    """Parse the first line of the given view, to get a tuple,
    which will contain the start token for a syntax test, and the closing token too if present.
    If the file doesn't contain syntax tests, both elements of the tuple will be `None`.
    """

    line = view.line(0)
    match = None
    if line.size() < 1000:  # no point checking longer lines as they are unlikely to match
        first_line = view.substr(line)
        match = re.match(r'^(\s*\S+)\s+SYNTAX TEST\s+"[^"]+"\s*(\S+)?$', first_line)
    if match is None:
        return (None, None)
    else:
        return (match.group(1), match.group(2))

def is_syntax_test_file(view):
    """Determine if the given view is a syntax test file or not.
    If the file has a name, check whether it begins with 'syntax_test_'.
    If it doesn't have a name / hasn't been saved yet, check the first line of the file.
    """

    name = view.file_name()
    if name is not None:
        name = path.basename(name)
        return name.startswith('syntax_test_')
    else:
        return get_syntax_test_tokens(view)[0] is not None

def get_details_of_test_assertion_line(view, pos):
    """Given a view and a character position, find:
    - the region of the line (3rd item in tuple aka `line_region`)
    - the comment marker (1st item in tuple aka `comment_marker_match`)
    - the assertion characters (2nd item in tuple aka `assertion_colrange`)
    """

    if not is_syntax_test_file(view):
        return AssertionLineDetails(None, None, None)
    tokens = get_syntax_test_tokens(view)
    if tokens[0] is None:
        return AssertionLineDetails(None, None, None)
    line_region = view.line(pos)
    line_text = view.substr(line_region)
    starts_with_comment_token = re.match(r'^\s*(' + re.escape(tokens[0]) + r')', line_text)
    assertion_colrange = None
    if starts_with_comment_token:
        assertion = re.match(r'\s*(?:(<-)|(\^+))', line_text[starts_with_comment_token.end():])
        if assertion:
            if assertion.group(1):
                assertion_colrange = (starts_with_comment_token.start(1),
                                      starts_with_comment_token.start(1))
            elif assertion.group(2):
                assertion_colrange = (starts_with_comment_token.end() + assertion.start(2),
                                      starts_with_comment_token.end() + assertion.end(2))

    return AssertionLineDetails(starts_with_comment_token, assertion_colrange, line_region)

def is_syntax_test_line(view, pos, must_contain_assertion):
    """Determine whether the line at the given character position is a syntax test line.
    It can optionally treat lines with comment markers but no assertion as a syntax test,
    useful for while the line is being written.
    """

    details = get_details_of_test_assertion_line(view, pos)
    if details.comment_marker_match:
        return not must_contain_assertion or details.assertion_colrange is not None
    return False

def get_details_of_line_being_tested(view):
    """Given a view, and starting from the cursor position, work upwards
    to find all syntax test lines that occur before the line that is being tested.
    Return a tuple containing a list of assertion line details,
    along with the region of the line being tested.
    """

    if not is_syntax_test_file(view):
        return (None, None)

    lines = []
    pos = view.sel()[0].begin()
    first_line = True
    while pos >= 0:
        details = get_details_of_test_assertion_line(view, pos)
        pos = details.line_region.begin() - 1
        if details.assertion_colrange:
            lines.append(details)
        elif not first_line or not details.comment_marker_match:
            break
        elif details.comment_marker_match:
            lines.append(details)
        first_line = False

    return (lines, details.line_region)

def find_common_scopes(scopes, skip_syntax_suffix):
    """Given a list of scopes, find the (partial) scopes that are common to each."""

    # skip the base scope i.e. `source.python`
    check_scopes = scopes[0].split()[1:]

    shared_scopes = []
    # stop as soon as at least one shared scope was found
    # or when there are no partial scopes left to check
    while not shared_scopes and check_scopes:
        if not skip_syntax_suffix:
            for check_scope in check_scopes:
                if all(sublime.score_selector(scope, check_scope) > 0 for scope in scopes):
                    shared_scopes.append(check_scope)

        # if no matches were found
        if not shared_scopes:
            # split off the last partial scope from each scope to check
            # i.e. `meta.function.parameters` becomes `meta.function`
            # if the scope to check doesn't contain any sub-scopes i.e. `meta`,
            # then drop it from the list of scopes to check
            check_scopes = [
                '.'.join(check_scope.split('.')[0:-1])
                for check_scope in check_scopes if '.' in check_scope
            ]
            skip_syntax_suffix = False

    return ' '.join(shared_scopes)

class AlignSyntaxTest(sublime_plugin.TextCommand):
    """Insert enough spaces so that the cursor will be immediately to the right of the
    previous line's last syntax test assertion.
    """

    def run(self, edit):
        cursor = self.view.sel()[0]
        details = get_details_of_test_assertion_line(self.view, cursor.begin())
        if not details.comment_marker_match:
            return

        # find the last test assertion column on the previous line
        details = get_details_of_test_assertion_line(self.view, details.line_region.begin() - 1)
        next_col = None
        if details.assertion_colrange is None:
            # the previous line wasn't a syntax test line, so instead
            # find the first non-whitespace char on the line being tested above
            for pos in range(details.line_region.begin(), details.line_region.end()):
                if self.view.substr(pos).strip() != '':
                    break
            next_col = self.view.rowcol(pos)[1]
        else:
            next_col = details.assertion_colrange[1]
        self.view.insert(edit, cursor.end(), ' ' * (
            next_col - self.view.rowcol(cursor.begin())[1]
        ))
        self.view.run_command('suggest_syntax_test')

class SuggestSyntaxTest(sublime_plugin.TextCommand):
    """Intelligently suggest where the syntax test assertions should be placed,
    based on the scopes on the line being tested, and where they change.
    """

    def run(self, edit, character='^'):
        """Available parameters:
        edit (sublime.Edit)
            The edit parameter from TextCommand.
        character (str) = '^'
            The character to insert when suggesting where the test assertions should go.
        """

        view = self.view
        view.replace(edit, view.sel()[0], character)
        insert_at = view.sel()[0].begin()
        if not is_syntax_test_file(view):
            return

        lines, line = get_details_of_line_being_tested(view)
        end_token = get_syntax_test_tokens(view)[1]
        # don't duplicate the end token if it is on the line but not selected
        if end_token is not None and view.sel()[0].end() == lines[0].line_region.end():
            end_token = ' ' + end_token
        else:
            end_token = ''

        scopes = []
        length = 0
        # find the following columns on the line to be tested where the scopes don't change
        test_at_start_of_comment = False
        col = view.rowcol(insert_at)[1]
        assertion_colrange = lines[0].assertion_colrange or (-1, -1)
        if assertion_colrange[0] == assertion_colrange[1]:
            col = assertion_colrange[1]
            test_at_start_of_comment = True
            lines = lines[1:]

        for pos in range(line.begin() + col, line.end() + 1):
            scope = view.scope_name(pos)
            if len(scopes) > 0:
                if scope != scopes[0]:
                    break
            else:
                scopes.append(scope)
            length += 1
            if test_at_start_of_comment:
                break

        # find the unique scopes at each existing assertion position
        if lines and not test_at_start_of_comment:
            col_start, col_end = lines[0].assertion_colrange
            for pos in range(line.begin() + col_start, line.begin() + col_end):
                scope = view.scope_name(pos)
                if scope not in scopes:
                    scopes.append(scope)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        suggest_suffix = prefs.get('syntax_test_suggest_scope_suffix', True)

        scope = find_common_scopes(scopes, not suggest_suffix)

        # delete the existing selection
        if not view.sel()[0].empty():
            view.erase(edit, view.sel()[0])

        view.insert(edit, insert_at, (character * length) + ' ' + scope + end_token)

        # move the selection to cover the added scope name,
        # so that the user can easily insert another ^ to extend the test
        view.sel().clear()
        view.sel().add(sublime.Region(
            insert_at + length,
            insert_at + length + len(' ' + scope + end_token)
        ))

class HighlightTestViewEventListener(sublime_plugin.ViewEventListener):
    def on_selection_modified_async(self):
        """When the selection changes, (re)move the highlight that shows where the current line's
        test assertions relate to.
        """

        if len(self.view.sel()) == 0:
            return
        cursor = self.view.sel()[0]
        highlight_only_cursor = False
        if cursor.empty():
            cursor = sublime.Region(cursor.begin(), cursor.end() + 1)
        else:
            highlight_only_cursor = re.match(r'^\^+$', self.view.substr(cursor)) is not None

        lines, line = get_details_of_line_being_tested(self.view)

        if not lines or not lines[0].assertion_colrange:
            self.view.erase_regions('current_syntax_test')
            return

        col_start, col_end = lines[0].assertion_colrange
        if highlight_only_cursor:
            col_start = self.view.rowcol(cursor.begin())[1]
            col_end = self.view.rowcol(cursor.end())[1]
        elif col_end == col_start:
            col_end += 1

        region = sublime.Region(line.begin() + col_start, line.begin() + col_end)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        scope = prefs.get('syntax_test_highlight_scope', 'text')
        styles = prefs.get('syntax_test_highlight_styles', ['DRAW_NO_FILL'])
        style_flags = 0
        # the following available add_region styles are taken from the API documentation:
        # http://www.sublimetext.com/docs/3/api_reference.html#sublime.View
        # unfortunately, the `sublime` module doesn't encapsulate them for easy reference
        # so we hardcode them here
        for style in styles:
            if style in [
                'DRAW_EMPTY', 'HIDE_ON_MINIMAP', 'DRAW_EMPTY_AS_OVERWRITE', 'DRAW_NO_FILL',
                'DRAW_NO_OUTLINE', 'DRAW_SOLID_UNDERLINE', 'DRAW_STIPPLED_UNDERLINE',
                'DRAW_SQUIGGLY_UNDERLINE', 'HIDDEN', 'PERSISTENT'
            ]:
                style_flags |= getattr(sublime, style)

        self.view.add_regions('current_syntax_test', [region], scope, '', style_flags)

    def on_query_context(self, key, operator, operand, match_all):
        """Respond to relevant syntax test keybinding contexts"""

        view = self.view
        # all contexts supported will have boolean results, so ignore regex operators
        if operator not in (sublime.OP_EQUAL, sublime.OP_NOT_EQUAL):
            return None

        keys = {
            "current_line_is_a_syntax_test":
                lambda: is_syntax_test_line(view, view.sel()[0].begin(), False),
            "file_contains_syntax_tests": lambda: is_syntax_test_file(view),
        }

        if key not in keys:
            return None
        else:
            result = keys[key]() == bool(operand)
            if operator == sublime.OP_NOT_EQUAL:
                result = not result
            return result
<commit_msg>Make current_line_is_a_syntax_test match_all-aware<commit_after>import sublime
import sublime_plugin
from os import path
import re
from collections import namedtuple

AssertionLineDetails = namedtuple(
    'AssertionLineDetails', ['comment_marker_match', 'assertion_colrange', 'line_region']
)

def get_syntax_test_tokens(view):
    """Parse the first line of the given view, to get a tuple,
    which will contain the start token for a syntax test, and the closing token too if present.
    If the file doesn't contain syntax tests, both elements of the tuple will be `None`.
    """

    line = view.line(0)
    match = None
    if line.size() < 1000:  # no point checking longer lines as they are unlikely to match
        first_line = view.substr(line)
        match = re.match(r'^(\s*\S+)\s+SYNTAX TEST\s+"[^"]+"\s*(\S+)?$', first_line)
    if match is None:
        return (None, None)
    else:
        return (match.group(1), match.group(2))

def is_syntax_test_file(view):
    """Determine if the given view is a syntax test file or not.
    If the file has a name, check whether it begins with 'syntax_test_'.
    If it doesn't have a name / hasn't been saved yet, check the first line of the file.
    """

    name = view.file_name()
    if name is not None:
        name = path.basename(name)
        return name.startswith('syntax_test_')
    else:
        return get_syntax_test_tokens(view)[0] is not None

def get_details_of_test_assertion_line(view, pos):
    """Given a view and a character position, find:
    - the region of the line (3rd item in tuple aka `line_region`)
    - the comment marker (1st item in tuple aka `comment_marker_match`)
    - the assertion characters (2nd item in tuple aka `assertion_colrange`)
    """

    if not is_syntax_test_file(view):
        return AssertionLineDetails(None, None, None)
    tokens = get_syntax_test_tokens(view)
    if tokens[0] is None:
        return AssertionLineDetails(None, None, None)
    line_region = view.line(pos)
    line_text = view.substr(line_region)
    starts_with_comment_token = re.match(r'^\s*(' + re.escape(tokens[0]) + r')', line_text)
    assertion_colrange = None
    if starts_with_comment_token:
        assertion = re.match(r'\s*(?:(<-)|(\^+))', line_text[starts_with_comment_token.end():])
        if assertion:
            if assertion.group(1):
                assertion_colrange = (starts_with_comment_token.start(1),
                                      starts_with_comment_token.start(1))
            elif assertion.group(2):
                assertion_colrange = (starts_with_comment_token.end() + assertion.start(2),
                                      starts_with_comment_token.end() + assertion.end(2))

    return AssertionLineDetails(starts_with_comment_token, assertion_colrange, line_region)

def is_syntax_test_line(view, pos, must_contain_assertion):
    """Determine whether the line at the given character position is a syntax test line.
    It can optionally treat lines with comment markers but no assertion as a syntax test,
    useful for while the line is being written.
    """

    details = get_details_of_test_assertion_line(view, pos)
    if details.comment_marker_match:
        return not must_contain_assertion or details.assertion_colrange is not None
    return False

def get_details_of_line_being_tested(view):
    """Given a view, and starting from the cursor position, work upwards
    to find all syntax test lines that occur before the line that is being tested.
    Return a tuple containing a list of assertion line details,
    along with the region of the line being tested.
    """

    if not is_syntax_test_file(view):
        return (None, None)

    lines = []
    pos = view.sel()[0].begin()
    first_line = True
    while pos >= 0:
        details = get_details_of_test_assertion_line(view, pos)
        pos = details.line_region.begin() - 1
        if details.assertion_colrange:
            lines.append(details)
        elif not first_line or not details.comment_marker_match:
            break
        elif details.comment_marker_match:
            lines.append(details)
        first_line = False

    return (lines, details.line_region)

def find_common_scopes(scopes, skip_syntax_suffix):
    """Given a list of scopes, find the (partial) scopes that are common to each."""

    # skip the base scope i.e. `source.python`
    check_scopes = scopes[0].split()[1:]

    shared_scopes = []
    # stop as soon as at least one shared scope was found
    # or when there are no partial scopes left to check
    while not shared_scopes and check_scopes:
        if not skip_syntax_suffix:
            for check_scope in check_scopes:
                if all(sublime.score_selector(scope, check_scope) > 0 for scope in scopes):
                    shared_scopes.append(check_scope)

        # if no matches were found
        if not shared_scopes:
            # split off the last partial scope from each scope to check
            # i.e. `meta.function.parameters` becomes `meta.function`
            # if the scope to check doesn't contain any sub-scopes i.e. `meta`,
            # then drop it from the list of scopes to check
            check_scopes = [
                '.'.join(check_scope.split('.')[0:-1])
                for check_scope in check_scopes if '.' in check_scope
            ]
            skip_syntax_suffix = False

    return ' '.join(shared_scopes)

class AlignSyntaxTest(sublime_plugin.TextCommand):
    """Insert enough spaces so that the cursor will be immediately to the right of the
    previous line's last syntax test assertion.
    """

    def run(self, edit):
        cursor = self.view.sel()[0]
        details = get_details_of_test_assertion_line(self.view, cursor.begin())
        if not details.comment_marker_match:
            return

        # find the last test assertion column on the previous line
        details = get_details_of_test_assertion_line(self.view, details.line_region.begin() - 1)
        next_col = None
        if details.assertion_colrange is None:
            # the previous line wasn't a syntax test line, so instead
            # find the first non-whitespace char on the line being tested above
            for pos in range(details.line_region.begin(), details.line_region.end()):
                if self.view.substr(pos).strip() != '':
                    break
            next_col = self.view.rowcol(pos)[1]
        else:
            next_col = details.assertion_colrange[1]
        self.view.insert(edit, cursor.end(), ' ' * (
            next_col - self.view.rowcol(cursor.begin())[1]
        ))
        self.view.run_command('suggest_syntax_test')

class SuggestSyntaxTest(sublime_plugin.TextCommand):
    """Intelligently suggest where the syntax test assertions should be placed,
    based on the scopes on the line being tested, and where they change.
    """

    def run(self, edit, character='^'):
        """Available parameters:
        edit (sublime.Edit)
            The edit parameter from TextCommand.
        character (str) = '^'
            The character to insert when suggesting where the test assertions should go.
        """

        view = self.view
        view.replace(edit, view.sel()[0], character)
        insert_at = view.sel()[0].begin()
        if not is_syntax_test_file(view):
            return

        lines, line = get_details_of_line_being_tested(view)
        end_token = get_syntax_test_tokens(view)[1]
        # don't duplicate the end token if it is on the line but not selected
        if end_token is not None and view.sel()[0].end() == lines[0].line_region.end():
            end_token = ' ' + end_token
        else:
            end_token = ''

        scopes = []
        length = 0
        # find the following columns on the line to be tested where the scopes don't change
        test_at_start_of_comment = False
        col = view.rowcol(insert_at)[1]
        assertion_colrange = lines[0].assertion_colrange or (-1, -1)
        if assertion_colrange[0] == assertion_colrange[1]:
            col = assertion_colrange[1]
            test_at_start_of_comment = True
            lines = lines[1:]

        for pos in range(line.begin() + col, line.end() + 1):
            scope = view.scope_name(pos)
            if len(scopes) > 0:
                if scope != scopes[0]:
                    break
            else:
                scopes.append(scope)
            length += 1
            if test_at_start_of_comment:
                break

        # find the unique scopes at each existing assertion position
        if lines and not test_at_start_of_comment:
            col_start, col_end = lines[0].assertion_colrange
            for pos in range(line.begin() + col_start, line.begin() + col_end):
                scope = view.scope_name(pos)
                if scope not in scopes:
                    scopes.append(scope)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        suggest_suffix = prefs.get('syntax_test_suggest_scope_suffix', True)

        scope = find_common_scopes(scopes, not suggest_suffix)

        # delete the existing selection
        if not view.sel()[0].empty():
            view.erase(edit, view.sel()[0])

        view.insert(edit, insert_at, (character * length) + ' ' + scope + end_token)

        # move the selection to cover the added scope name,
        # so that the user can easily insert another ^ to extend the test
        view.sel().clear()
        view.sel().add(sublime.Region(
            insert_at + length,
            insert_at + length + len(' ' + scope + end_token)
        ))

class HighlightTestViewEventListener(sublime_plugin.ViewEventListener):
    def on_selection_modified_async(self):
        """When the selection changes, (re)move the highlight that shows where the current line's
        test assertions relate to.
        """

        if len(self.view.sel()) == 0:
            return
        cursor = self.view.sel()[0]
        highlight_only_cursor = False
        if cursor.empty():
            cursor = sublime.Region(cursor.begin(), cursor.end() + 1)
        else:
            highlight_only_cursor = re.match(r'^\^+$', self.view.substr(cursor)) is not None

        lines, line = get_details_of_line_being_tested(self.view)

        if not lines or not lines[0].assertion_colrange:
            self.view.erase_regions('current_syntax_test')
            return

        col_start, col_end = lines[0].assertion_colrange
        if highlight_only_cursor:
            col_start = self.view.rowcol(cursor.begin())[1]
            col_end = self.view.rowcol(cursor.end())[1]
        elif col_end == col_start:
            col_end += 1

        region = sublime.Region(line.begin() + col_start, line.begin() + col_end)

        prefs = sublime.load_settings('PackageDev.sublime-settings')
        scope = prefs.get('syntax_test_highlight_scope', 'text')
        styles = prefs.get('syntax_test_highlight_styles', ['DRAW_NO_FILL'])
        style_flags = 0
        # the following available add_region styles are taken from the API documentation:
        # http://www.sublimetext.com/docs/3/api_reference.html#sublime.View
        # unfortunately, the `sublime` module doesn't encapsulate them for easy reference
        # so we hardcode them here
        for style in styles:
            if style in [
                'DRAW_EMPTY', 'HIDE_ON_MINIMAP', 'DRAW_EMPTY_AS_OVERWRITE', 'DRAW_NO_FILL',
                'DRAW_NO_OUTLINE', 'DRAW_SOLID_UNDERLINE', 'DRAW_STIPPLED_UNDERLINE',
                'DRAW_SQUIGGLY_UNDERLINE', 'HIDDEN', 'PERSISTENT'
            ]:
                style_flags |= getattr(sublime, style)

        self.view.add_regions('current_syntax_test', [region], scope, '', style_flags)

    def on_query_context(self, key, operator, operand, match_all):
        """Respond to relevant syntax test keybinding contexts"""

        view = self.view
        # all contexts supported will have boolean results, so ignore regex operators
        if operator not in (sublime.OP_EQUAL, sublime.OP_NOT_EQUAL):
            return None

        def current_line_is_a_syntax_test():
            results = (is_syntax_test_line(view, reg.begin(), False)
                       for reg in view.sel())
            aggregator = all if match_all else any
            return aggregator(results)

        keys = {
            "current_line_is_a_syntax_test": current_line_is_a_syntax_test,
            "file_contains_syntax_tests": lambda: is_syntax_test_file(view),
        }

        if key not in keys:
            return None
        else:
            result = keys[key]() == bool(operand)
            if operator == sublime.OP_NOT_EQUAL:
                result = not result
            return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Script by Ben Limmer
# https://github.com/l1m5
#
# This simple Python script will combine all the host files you provide
# as sources into one, unique host file to keep you internet browsing happy.

# Making Python 2 compatible with Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import platform
import re
import string
import subprocess
import sys
import tempfile
# zip files are not used actually, support deleted
# StringIO is not needed in Python 3
# Python 3 works differently with urlopen

# Supporting urlopen in Python 2 and Python 3
try:
	from urllib.parse import urlparse, urlencode
	from urllib.request import urlopen, Request
	from urllib.error import HTTPError
except ImportError:
	from urlparse import urlparse
	from urllib import urlencode
	from urllib2 import urlopen, Request, HTTPError

# This function handles both Python 2 and Python 3
def getFileByUrl(url):
	try:
		f = urlopen(url)
		return f.read().decode("UTF-8")
	except:
		print ("Problem getting file: ", url);
		# raise

# In Python 3   "print" is a function, braces are added everywhere

# Detecting Python 3 for version-dependent implementations
Python3=False;
cur_version = sys.version_info
if cur_version >= (3, 0):
	Python3=True;

# This function works in both Python 2 and Python 3
def myInput(msg=""):
	if Python3:
		return input(msg);
	else:
		return raw_input(msg);

# Cross-python writing function
def writeData(f, data):
	if Python3:
		f.write(bytes(data, 'UTF-8'))
	else:
		f.write(str(data).encode('UTF-8'))

# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
DATA_PATH = os.path.join(BASEDIR_PATH, 'data')
DATA_FILENAMES = 'hosts'
UPDATE_URL_FILENAME = 'update.info'
SOURCES = os.listdir(DATA_PATH)
README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md')
README_FILE = os.path.join(BASEDIR_PATH, 'readme.md')
TARGET_HOST = '0.0.0.0'

# Exclusions
EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end

# Common domains to exclude
COMMON_EXCLUSIONS = ['hulu.com']

# Global vars
exclusionRegexs = []
numberOfRules = 0

def main():
	promptForUpdate()
	promptForExclusions()
	mergeFile = createInitialFile()
	finalFile = removeDups(mergeFile)
	finalizeFile(finalFile)
	updateReadme(numberOfRules)
	printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.')

	promptForMove(finalFile)

# Prompt the User
def promptForUpdate():
	response = query_yes_no("Do you want to update all data sources?")
	if (response == "yes"):
		updateAllSources()
	else:
		print ('OK, we\'ll stick with what we\'ve  got locally.')

def promptForExclusions():
	response = query_yes_no("Do you want to exclude any domains?\n" +
							"For example, hulu.com video streaming must be able to access " +
							"its tracking and ad servers in order to play video.")
	if (response == "yes"):
		displayExclusionOptions()
	else:
		print ('OK, we won\'t exclude any domains.')

def promptForMoreCustomExclusions():
	response = query_yes_no("Do you have more domains you want to enter?")
	if (response == "yes"):
		return True
	else:
		return False

def promptForMove(finalFile):
  response = query_yes_no("Do you want to replace your existing hosts file with the newly generated file?")
  if (response == "yes"):
    moveHostsFileIntoPlace(finalFile)
  else:
    return False
# End Prompt the User

# Exclusion logic
def displayExclusionOptions():
	for exclusionOption in COMMON_EXCLUSIONS:
		response = query_yes_no("Do you want to exclude the domain " + exclusionOption + " ?")
		if (response == "yes"):
			excludeDomain(exclusionOption)
		else:
			continue
	response = query_yes_no("Do you want to exclude any other domains?")
	if (response == "yes"):
		gatherCustomExclusions()

def gatherCustomExclusions():
	while True:
		# Cross-python Input
		domainFromUser = myInput("Enter the domain you want to exclude (e.g. facebook.com): ")
		if (isValidDomainFormat(domainFromUser)):
			excludeDomain(domainFromUser)
		if (promptForMoreCustomExclusions() == False):
			return

def excludeDomain(domain):
	exclusionRegexs.append(re.compile(EXCLUSION_PATTERN + domain))

def matchesExclusions(strippedRule):
	strippedDomain = strippedRule.split()[1]
	for exclusionRegex in exclusionRegexs:
		if exclusionRegex.search(strippedDomain):
			return True
	return False
# End Exclusion Logic

# Update Logic
def updateAllSources():
	for source in SOURCES:
		updateURL = getUpdateURLFromFile(source)
		if (updateURL == None):
			continue;
		print ('Updating source ' + source + ' from ' + updateURL)
		# Cross-python call
		updatedFile = getFileByUrl(updateURL);
		updatedFile = updatedFile.replace('\r', '') #get rid of carriage-return symbols

		# This is cross-python code
		dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'wb')
		writeData(dataFile, updatedFile);
		dataFile.close()

def getUpdateURLFromFile(source):
	pathToUpdateFile = os.path.join(DATA_PATH, source, UPDATE_URL_FILENAME)
	if os.path.exists(pathToUpdateFile):
		updateFile = open(pathToUpdateFile, 'r')
		retURL = updateFile.readline().strip()
		updateFile.close()
	else:
		retURL = None
		printFailure('Warning: Can\'t find the update file for source ' + source + '\n' +
					 'Make sure that there\'s a file at ' + pathToUpdateFile)
	return retURL
# End Update Logic

# File Logic
def createInitialFile():
	mergeFile = tempfile.NamedTemporaryFile()
	for source in SOURCES:
		curFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'r')
		#Done in a cross-python way
		writeData(mergeFile, '\n# Begin ' + source + '\n')
		writeData(mergeFile, curFile.read())
		writeData(mergeFile, '\n# End ' + source + '\n')

	return mergeFile

def removeDups(mergeFile):
	global numberOfRules

    # Another mode is required to read and write the file in Python 3
	finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+b')
	mergeFile.seek(0) # reset file pointer

	hostnames = set()
	hostnames.add("localhost")
	for line in mergeFile.readlines():
        # Explicit encoding
		line = line.decode("UTF-8")
		# Testing the first character doesn't require startswith
		if line[0] == '#' or re.match(r'^\s*$', line[0]):
			# Cross-python write
			writeData(finalFile, line)
			continue

		strippedRule = stripRule(line) #strip comments
		if matchesExclusions(strippedRule):
			continue
		hostname, normalizedRule = normalizeRule(strippedRule) # normalize rule

		if normalizedRule and (hostname not in hostnames):
			writeData(finalFile, normalizedRule)
			hostnames.add(hostname)
			numberOfRules += 1

	mergeFile.close()

	return finalFile

def normalizeRule(rule):
	result = re.search(r'^[ \t]*(\d+\.\d+\.\d+\.\d+)\s+([\w\.-]+)(.*)',rule)
	if result:
		target, hostname, suffix = result.groups()
		hostname = hostname.lower() # explicitly lowercase hostname
		if suffix is not '':
			# add suffix as comment only, not as a separate host
			return hostname, "%s %s #%s\n" % (TARGET_HOST, hostname, suffix)
		else:
			return hostname, "%s %s\n" % (TARGET_HOST, hostname)
	print ('==>%s<==' % rule)
	return None, None

def finalizeFile(finalFile):
	writeOpeningHeader(finalFile)
	finalFile.close()

# Some sources put comments around their rules, for accuracy we need to strip them
# the comments are preserved in the output hosts file
def stripRule(line):
	splitLine = line.split()
	if (len(splitLine) < 2) :
        # This is due to the diffrences between bytes and string type in Python 3
		printFailure('A line in the hostfile is going to cause problems because it is nonstandard\n' +
					 'The line reads ' + str(line) + ' please check your data files. Maybe you have a comment without a #?')
		sys.exit()
	return splitLine[0] + ' ' + splitLine[1]

def writeOpeningHeader(finalFile):
	global numberOfRules
	finalFile.seek(0) #reset file pointer
	fileContents = finalFile.read(); #save content
	finalFile.seek(0) #write at the top
	writeData(finalFile, '# This file is a merged collection of hosts from reputable sources,\n')
	writeData(finalFile, '# with a dash of crowd sourcing via Github\n#\n')
	writeData(finalFile, '# Project home page: https://github.com/StevenBlack/hosts\n#\n')
	writeData(finalFile, '# Current sources:\n')
	for source in SOURCES:
		writeData(finalFile, '#    ' + source + '\n')
	writeData(finalFile, '#\n')
	writeData(finalFile, '# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n')
	writeData(finalFile, '# ===============================================================\n')
	writeData(finalFile, '\n')
	writeData(finalFile, '127.0.0.1 localhost\n')
	writeData(finalFile, '\n')

	preamble = os.path.join(BASEDIR_PATH, "myhosts");
	if os.path.isfile(preamble):
		with open(preamble, "r") as f:
			writeData(finalFile, f.read());

	finalFile.write(fileContents)

def updateReadme(numberOfRules):
	with open(README_FILE, "wt") as out:
		for line in open(README_TEMPLATE):
			out.write(line.replace('@NUM_ENTRIES@', "{:,}".format( numberOfRules )))

def moveHostsFileIntoPlace(finalFile):
	if (os.name == 'posix'):
		print ('Moving the file requires administrative privileges. You might need to enter your password.')
		if(subprocess.call(["/usr/bin/sudo", "cp", os.path.abspath(finalFile.name), "/etc/hosts"])):
			printFailure("Moving the file failed.")
		print ('Flushing the DNS Cache to utilize new hosts file...')
		if (platform.system() == 'Darwin'):
			if(subprocess.call(["/usr/bin/sudo", "killall", "-HUP", "mDNSResponder"])):
				printFailure("Flushing the DNS Cache failed.")
		else:
			if os.path.isfile("/etc/rc.d/init.d/nscd"):
				if(subprocess.call(["/usr/bin/sudo", "/etc/rc.d/init.d/nscd", "restart"])):
					printFailure("Flushing the DNS Cache failed.")
	elif (os.name == 'nt'):
		print ('Automatically moving the hosts file in place is not yet supported.')
		print ('Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts')

# End File Logic

# Helper Functions
## {{{ http://code.activestate.com/recipes/577058/ (r2)
def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is one of "yes" or "no".
    """
    valid = {"yes":"yes",   "y":"yes",  "ye":"yes",
             "no":"no",     "n":"no"}
    if default == None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while 1:
        sys.stdout.write(colorize(question, colors.PROMPT) + prompt)
        # Changed to be cross-python
        choice = myInput().lower()
        if default is not None and choice == '':
            return default
        elif choice in valid.keys():
            return valid[choice]
        else:
            printFailure("Please respond with 'yes' or 'no' "\
                             "(or 'y' or 'n').\n")
## end of http://code.activestate.com/recipes/577058/ }}}

def isValidDomainFormat(domain):
	if (domain == ''):
		print ("You didn\'t enter a domain. Try again.")
		return False
	domainRegex = re.compile("www\d{0,3}[.]|https?")
	if (domainRegex.match(domain)):
		print ("The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again.")
		return False
	else:
		return True

# Colors
class colors:
    PROMPT 	= '\033[94m'
    SUCCESS = '\033[92m'
    FAIL 	= '\033[91m'
    ENDC 	= '\033[0m'

def colorize(text, color):
	return color + text + colors.ENDC

def printSuccess(text):
	print (colorize(text, colors.SUCCESS))

def printFailure(text):
	print (colorize(text, colors.FAIL))
# End Helper Functions

if __name__ == "__main__":
	main()
<commit_msg>Delete .DS_Store from Data<commit_after>#!/usr/bin/env python

# Script by Ben Limmer
# https://github.com/l1m5
#
# This simple Python script will combine all the host files you provide
# as sources into one, unique host file to keep you internet browsing happy.

# Making Python 2 compatible with Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import platform
import re
import string
import subprocess
import sys
import tempfile
# zip files are not used actually, support deleted
# StringIO is not needed in Python 3
# Python 3 works differently with urlopen

# Supporting urlopen in Python 2 and Python 3
try:
	from urllib.parse import urlparse, urlencode
	from urllib.request import urlopen, Request
	from urllib.error import HTTPError
except ImportError:
	from urlparse import urlparse
	from urllib import urlencode
	from urllib2 import urlopen, Request, HTTPError

# This function handles both Python 2 and Python 3
def getFileByUrl(url):
	try:
		f = urlopen(url)
		return f.read().decode("UTF-8")
	except:
		print ("Problem getting file: ", url);
		# raise

# In Python 3   "print" is a function, braces are added everywhere

# Detecting Python 3 for version-dependent implementations
Python3=False;
cur_version = sys.version_info
if cur_version >= (3, 0):
	Python3=True;

# This function works in both Python 2 and Python 3
def myInput(msg=""):
	if Python3:
		return input(msg);
	else:
		return raw_input(msg);

# Cross-python writing function
def writeData(f, data):
	if Python3:
		f.write(bytes(data, 'UTF-8'))
	else:
		f.write(str(data).encode('UTF-8'))

# Project Settings
BASEDIR_PATH = os.path.dirname(os.path.realpath(__file__))
DATA_PATH = os.path.join(BASEDIR_PATH, 'data')
DATA_FILENAMES = 'hosts'
UPDATE_URL_FILENAME = 'update.info'
SOURCES = os.listdir(DATA_PATH)
README_TEMPLATE = os.path.join(BASEDIR_PATH, 'readme_template.md')
README_FILE = os.path.join(BASEDIR_PATH, 'readme.md')
TARGET_HOST = '0.0.0.0'
DS_STORE = BASEDIR_PATH + '/data/.DS_Store'
if os.path.isfile(DS_STORE):
	os.remove(DS_STORE)

# Exclusions
EXCLUSION_PATTERN = '([a-zA-Z\d-]+\.){0,}' #append domain the end

# Common domains to exclude
COMMON_EXCLUSIONS = ['hulu.com']

# Global vars
exclusionRegexs = []
numberOfRules = 0

def main():
	promptForUpdate()
	promptForExclusions()
	mergeFile = createInitialFile()
	finalFile = removeDups(mergeFile)
	finalizeFile(finalFile)
	updateReadme(numberOfRules)
	printSuccess('Success! Your shiny new hosts file has been prepared.\nIt contains ' + "{:,}".format( numberOfRules ) + ' unique entries.')

	promptForMove(finalFile)

# Prompt the User
def promptForUpdate():
	response = query_yes_no("Do you want to update all data sources?")
	if (response == "yes"):
		updateAllSources()
	else:
		print ('OK, we\'ll stick with what we\'ve  got locally.')

def promptForExclusions():
	response = query_yes_no("Do you want to exclude any domains?\n" +
							"For example, hulu.com video streaming must be able to access " +
							"its tracking and ad servers in order to play video.")
	if (response == "yes"):
		displayExclusionOptions()
	else:
		print ('OK, we won\'t exclude any domains.')

def promptForMoreCustomExclusions():
	response = query_yes_no("Do you have more domains you want to enter?")
	if (response == "yes"):
		return True
	else:
		return False

def promptForMove(finalFile):
  response = query_yes_no("Do you want to replace your existing hosts file with the newly generated file?")
  if (response == "yes"):
    moveHostsFileIntoPlace(finalFile)
  else:
    return False
# End Prompt the User

# Exclusion logic
def displayExclusionOptions():
	for exclusionOption in COMMON_EXCLUSIONS:
		response = query_yes_no("Do you want to exclude the domain " + exclusionOption + " ?")
		if (response == "yes"):
			excludeDomain(exclusionOption)
		else:
			continue
	response = query_yes_no("Do you want to exclude any other domains?")
	if (response == "yes"):
		gatherCustomExclusions()

def gatherCustomExclusions():
	while True:
		# Cross-python Input
		domainFromUser = myInput("Enter the domain you want to exclude (e.g. facebook.com): ")
		if (isValidDomainFormat(domainFromUser)):
			excludeDomain(domainFromUser)
		if (promptForMoreCustomExclusions() == False):
			return

def excludeDomain(domain):
	exclusionRegexs.append(re.compile(EXCLUSION_PATTERN + domain))

def matchesExclusions(strippedRule):
	strippedDomain = strippedRule.split()[1]
	for exclusionRegex in exclusionRegexs:
		if exclusionRegex.search(strippedDomain):
			return True
	return False
# End Exclusion Logic

# Update Logic
def updateAllSources():
	for source in SOURCES:
		updateURL = getUpdateURLFromFile(source)
		if (updateURL == None):
			continue;
		print ('Updating source ' + source + ' from ' + updateURL)
		# Cross-python call
		updatedFile = getFileByUrl(updateURL);
		updatedFile = updatedFile.replace('\r', '') #get rid of carriage-return symbols

		# This is cross-python code
		dataFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'wb')
		writeData(dataFile, updatedFile);
		dataFile.close()

def getUpdateURLFromFile(source):
	pathToUpdateFile = os.path.join(DATA_PATH, source, UPDATE_URL_FILENAME)
	if os.path.exists(pathToUpdateFile):
		updateFile = open(pathToUpdateFile, 'r')
		retURL = updateFile.readline().strip()
		updateFile.close()
	else:
		retURL = None
		printFailure('Warning: Can\'t find the update file for source ' + source + '\n' +
					 'Make sure that there\'s a file at ' + pathToUpdateFile)
	return retURL
# End Update Logic

# File Logic
def createInitialFile():
	mergeFile = tempfile.NamedTemporaryFile()
	for source in SOURCES:
		curFile = open(os.path.join(DATA_PATH, source, DATA_FILENAMES), 'r')
		#Done in a cross-python way
		writeData(mergeFile, '\n# Begin ' + source + '\n')
		writeData(mergeFile, curFile.read())
		writeData(mergeFile, '\n# End ' + source + '\n')

	return mergeFile

def removeDups(mergeFile):
	global numberOfRules

    # Another mode is required to read and write the file in Python 3
	finalFile = open(os.path.join(BASEDIR_PATH, 'hosts'), 'r+b')
	mergeFile.seek(0) # reset file pointer

	hostnames = set()
	hostnames.add("localhost")
	for line in mergeFile.readlines():
        # Explicit encoding
		line = line.decode("UTF-8")
		# Testing the first character doesn't require startswith
		if line[0] == '#' or re.match(r'^\s*$', line[0]):
			# Cross-python write
			writeData(finalFile, line)
			continue

		strippedRule = stripRule(line) #strip comments
		if matchesExclusions(strippedRule):
			continue
		hostname, normalizedRule = normalizeRule(strippedRule) # normalize rule

		if normalizedRule and (hostname not in hostnames):
			writeData(finalFile, normalizedRule)
			hostnames.add(hostname)
			numberOfRules += 1

	mergeFile.close()

	return finalFile

def normalizeRule(rule):
	result = re.search(r'^[ \t]*(\d+\.\d+\.\d+\.\d+)\s+([\w\.-]+)(.*)',rule)
	if result:
		target, hostname, suffix = result.groups()
		hostname = hostname.lower() # explicitly lowercase hostname
		if suffix is not '':
			# add suffix as comment only, not as a separate host
			return hostname, "%s %s #%s\n" % (TARGET_HOST, hostname, suffix)
		else:
			return hostname, "%s %s\n" % (TARGET_HOST, hostname)
	print ('==>%s<==' % rule)
	return None, None

def finalizeFile(finalFile):
	writeOpeningHeader(finalFile)
	finalFile.close()

# Some sources put comments around their rules, for accuracy we need to strip them
# the comments are preserved in the output hosts file
def stripRule(line):
	splitLine = line.split()
	if (len(splitLine) < 2) :
        # This is due to the diffrences between bytes and string type in Python 3
		printFailure('A line in the hostfile is going to cause problems because it is nonstandard\n' +
					 'The line reads ' + str(line) + ' please check your data files. Maybe you have a comment without a #?')
		sys.exit()
	return splitLine[0] + ' ' + splitLine[1]

def writeOpeningHeader(finalFile):
	global numberOfRules
	finalFile.seek(0) #reset file pointer
	fileContents = finalFile.read(); #save content
	finalFile.seek(0) #write at the top
	writeData(finalFile, '# This file is a merged collection of hosts from reputable sources,\n')
	writeData(finalFile, '# with a dash of crowd sourcing via Github\n#\n')
	writeData(finalFile, '# Project home page: https://github.com/StevenBlack/hosts\n#\n')
	writeData(finalFile, '# Current sources:\n')
	for source in SOURCES:
		writeData(finalFile, '#    ' + source + '\n')
	writeData(finalFile, '#\n')
	writeData(finalFile, '# Merging these sources produced ' + "{:,}".format( numberOfRules ) + ' unique entries\n')
	writeData(finalFile, '# ===============================================================\n')
	writeData(finalFile, '\n')
	writeData(finalFile, '127.0.0.1 localhost\n')
	writeData(finalFile, '\n')

	preamble = os.path.join(BASEDIR_PATH, "myhosts");
	if os.path.isfile(preamble):
		with open(preamble, "r") as f:
			writeData(finalFile, f.read());

	finalFile.write(fileContents)

def updateReadme(numberOfRules):
	with open(README_FILE, "wt") as out:
		for line in open(README_TEMPLATE):
			out.write(line.replace('@NUM_ENTRIES@', "{:,}".format( numberOfRules )))

def moveHostsFileIntoPlace(finalFile):
	if (os.name == 'posix'):
		print ('Moving the file requires administrative privileges. You might need to enter your password.')
		if(subprocess.call(["/usr/bin/sudo", "cp", os.path.abspath(finalFile.name), "/etc/hosts"])):
			printFailure("Moving the file failed.")
		print ('Flushing the DNS Cache to utilize new hosts file...')
		if (platform.system() == 'Darwin'):
			if(subprocess.call(["/usr/bin/sudo", "killall", "-HUP", "mDNSResponder"])):
				printFailure("Flushing the DNS Cache failed.")
		else:
			if os.path.isfile("/etc/rc.d/init.d/nscd"):
				if(subprocess.call(["/usr/bin/sudo", "/etc/rc.d/init.d/nscd", "restart"])):
					printFailure("Flushing the DNS Cache failed.")
	elif (os.name == 'nt'):
		print ('Automatically moving the hosts file in place is not yet supported.')
		print ('Please move the generated file to %SystemRoot%\system32\drivers\etc\hosts')

# End File Logic

# Helper Functions
## {{{ http://code.activestate.com/recipes/577058/ (r2)
def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is one of "yes" or "no".
    """
    valid = {"yes":"yes",   "y":"yes",  "ye":"yes",
             "no":"no",     "n":"no"}
    if default == None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while 1:
        sys.stdout.write(colorize(question, colors.PROMPT) + prompt)
        # Changed to be cross-python
        choice = myInput().lower()
        if default is not None and choice == '':
            return default
        elif choice in valid.keys():
            return valid[choice]
        else:
            printFailure("Please respond with 'yes' or 'no' "\
                             "(or 'y' or 'n').\n")
## end of http://code.activestate.com/recipes/577058/ }}}

def isValidDomainFormat(domain):
	if (domain == ''):
		print ("You didn\'t enter a domain. Try again.")
		return False
	domainRegex = re.compile("www\d{0,3}[.]|https?")
	if (domainRegex.match(domain)):
		print ("The domain " + domain + " is not valid. Do not include www.domain.com or http(s)://domain.com. Try again.")
		return False
	else:
		return True

# Colors
class colors:
    PROMPT 	= '\033[94m'
    SUCCESS = '\033[92m'
    FAIL 	= '\033[91m'
    ENDC 	= '\033[0m'

def colorize(text, color):
	return color + text + colors.ENDC

def printSuccess(text):
	print (colorize(text, colors.SUCCESS))

def printFailure(text):
	print (colorize(text, colors.FAIL))
# End Helper Functions

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line skeleton application for Calendar API.
Usage:
  $ python sample.py

You can also get help on all the command-line flags the program understands
by running:

  $ python sample.py --help

"""

import argparse
import httplib2
import os
import sys
import MovieFetcher
import datetime

from apiclient import discovery
from oauth2client import file
from oauth2client import client
from oauth2client import tools

from prettytable import PrettyTable

# Parser for command-line arguments.
parser = argparse.ArgumentParser(
    description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter,
    parents=[tools.argparser])

# CLIENT_SECRETS is name of a file containing the OAuth 2.0 information for this
# application, including client_id and client_secret. You can see the Client ID
# and Client secret on the APIs page in the Cloud Console:
# <https://cloud.google.com/console#/project/966749676004/apiui>
CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = client.flow_from_clientsecrets(CLIENT_SECRETS,
  scope=[
      'https://www.googleapis.com/auth/calendar',
      'https://www.googleapis.com/auth/calendar.readonly',
    ],
    message=tools.message_if_missing(CLIENT_SECRETS))

def get_service():
    """Returns a service object used to interact with the calendar."""
    storage = file.Storage('sample.dat')
    credentials = storage.get()
    flags = parser.parse_args(sys.argv[1:])
    if credentials is None or credentials.invalid:
      credentials = tools.run_flow(FLOW, storage, flags)

    # Create an httplib2.Http object to handle our HTTP requests and authorize it
    # with our good Credentials.
    http = httplib2.Http()
    http = credentials.authorize(http)

    # Construct the service object for the interacting with the Calendar API.
    service = discovery.build('calendar', 'v3', http=http)
    return service

def get_calendars(service):
    """Returns a list of calenders, each represented by a dictionary."""
    return service.calendarList().list().execute()['items']

def get_table(fields):
    table=PrettyTable(field_names=fields)
    for key in table.align:
        table.align[key] = "l"
    return table

def print_calendars(fields=[u'id', u'description', u'summary', u'timeZone']):
    """Prints a the key features of a calendar list."""
    service=get_service()
    calendars=get_calendars(service)

    table=get_table(fields)

    for cal in calendars:
        row=[]
        for field in fields:
            row.append(cal.get(field, None))
        table.add_row(row)
    print table

def get_events(calendarId):
    service=get_service()
    return service.events().list(calendarId=calendarId, maxResults=5000).execute()['items']

def print_events(events):
    fields=[u'id', u'summary', u'start', u'end']
    table=PrettyTable(field_names=fields)
    for key in table.align:
        table.align[key] = "l"
    for event in events:
        row=[]
        for field in fields:
            row.append(event.get(field, None))
        table.add_row(row)
    print table

def delete_events(service, calendar_id):
    for event in get_events(calendar_id):
        service.events().delete(calendarId=calendar_id, eventId=event['id']).execute()

def main(argv):
  service=get_service()

  try:
    sderot_calendar_id="matan.name_55j9srv12aamsve51u2vvm9cuk@group.calendar.google.com"
    #events=service.events().list(calendarId=tarbut_calendar_id).execute()
    #for event in events['items']:
    #    print event['summary'], event.keys()

    #print service.events().quickAdd(calendarId=sderot_calendar_id, text="Seret at Sderot! November 5th, 2013 13:00-14:00", sendNotifications=None).execute()
    #print get_calendars(service)
    #print_calendars()
    schedule=MovieFetcher.Schedule()
    events_already_in_calendar=get_events(sderot_calendar_id)

    #for screening in schedule.screenings:

     #   print screening
        #
        #event = {
        #    'summary': screening.movie.title,
        #    'location': 'הדגל 4, שדרות, ישראל',
        #    'description': screening.movie.description,
        #    'start': {
        #        'dateTime': screening.date.strftime("%Y-%m-%dT%H:%M:%S.000+02:00") #2011-06-03T10:00:00.000-07:00
        #    },
        #    'end': {
        #        'dateTime': (screening.date+datetime.timedelta(0, screening.movie.duration*60)).strftime("%Y-%m-%dT%H:%M:%S.00+02:00")
        #    },
        #
        #    }
        #
        #print event
        #created_event = service.events().insert(calendarId=sderot_calendar_id, body=event).execute()

    print_events(get_events(sderot_calendar_id))
    print(schedule)

  except client.AccessTokenRefreshError:
    print ("The credentials have been revoked or expired, please re-run"
      "the application to re-authorize")

if __name__ == '__main__':
  main(sys.argv)

<commit_msg>Rearranged code<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2013 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Command-line skeleton application for Calendar API.
Usage:
  $ python sample.py

You can also get help on all the command-line flags the program understands
by running:

  $ python sample.py --help

"""

import argparse
import httplib2
import os
import sys
import MovieFetcher
import datetime

from apiclient import discovery
from oauth2client import file
from oauth2client import client
from oauth2client import tools

from prettytable import PrettyTable

# Parser for command-line arguments.
parser = argparse.ArgumentParser(
    description=__doc__,
    formatter_class=argparse.RawDescriptionHelpFormatter,
    parents=[tools.argparser])

# CLIENT_SECRETS is name of a file containing the OAuth 2.0 information for this
# application, including client_id and client_secret. You can see the Client ID
# and Client secret on the APIs page in the Cloud Console:
# <https://cloud.google.com/console#/project/966749676004/apiui>
CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), 'client_secrets.json')

# Set up a Flow object to be used for authentication.
# Add one or more of the following scopes. PLEASE ONLY ADD THE SCOPES YOU
# NEED. For more information on using scopes please see
# <https://developers.google.com/+/best-practices>.
FLOW = client.flow_from_clientsecrets(CLIENT_SECRETS,
  scope=[
      'https://www.googleapis.com/auth/calendar',
      'https://www.googleapis.com/auth/calendar.readonly',
    ],
    message=tools.message_if_missing(CLIENT_SECRETS))

def get_service():
    """Returns a service object used to interact with the calendar."""
    storage = file.Storage('sample.dat')
    credentials = storage.get()
    flags = parser.parse_args(sys.argv[1:])
    if credentials is None or credentials.invalid:
      credentials = tools.run_flow(FLOW, storage, flags)

    # Create an httplib2.Http object to handle our HTTP requests and authorize it
    # with our good Credentials.
    http = httplib2.Http()
    http = credentials.authorize(http)

    # Construct the service object for the interacting with the Calendar API.
    service = discovery.build('calendar', 'v3', http=http)
    return service

def get_calendars(service):
    """Returns a list of calenders, each represented by a dictionary."""
    return service.calendarList().list().execute()['items']

def get_table(fields):
    table=PrettyTable(field_names=fields)
    for key in table.align:
        table.align[key] = "l"
    return table

def print_calendars(fields=[u'id', u'description', u'summary', u'timeZone']):
    """Prints a the key features of a calendar list."""
    service=get_service()
    calendars=get_calendars(service)

    table=get_table(fields)

    for cal in calendars:
        row=[]
        for field in fields:
            row.append(cal.get(field, None))
        table.add_row(row)
    print table

def get_events(calendarId):
    service=get_service()
    return service.events().list(calendarId=calendarId, maxResults=5000).execute()['items']

def print_events(events):
    fields=[u'id', u'summary', u'start', u'end']
    table=PrettyTable(field_names=fields)
    for key in table.align:
        table.align[key] = "l"
    for event in events:
        row=[]
        for field in fields:
            row.append(event.get(field, None))
        table.add_row(row)
    print table

def delete_events(service, calendar_id):
    for event in get_events(calendar_id):
        service.events().delete(calendarId=calendar_id, eventId=event['id']).execute()

def main(argv):
  service=get_service()

  try:
    sderot_calendar_id="matan.name_55j9srv12aamsve51u2vvm9cuk@group.calendar.google.com"
    schedule=MovieFetcher.Schedule()
    events_already_in_calendar=get_events(sderot_calendar_id)

    #for screening in schedule.screenings:

     #   print screening
        #
        #event = {
        #    'summary': screening.movie.title,
        #    'location': 'הדגל 4, שדרות, ישראל',
        #    'description': screening.movie.description,
        #    'start': {
        #        'dateTime': screening.date.strftime("%Y-%m-%dT%H:%M:%S.000+02:00") #2011-06-03T10:00:00.000-07:00
        #    },
        #    'end': {
        #        'dateTime': (screening.date+datetime.timedelta(0, screening.movie.duration*60)).strftime("%Y-%m-%dT%H:%M:%S.00+02:00")
        #    },
        #
        #    }
        #
        #print event
        #created_event = service.events().insert(calendarId=sderot_calendar_id, body=event).execute()

    print_events(get_events(sderot_calendar_id))
    print(schedule)

  except client.AccessTokenRefreshError:
    print ("The credentials have been revoked or expired, please re-run"
      "the application to re-authorize")

if __name__ == '__main__':
  main(sys.argv)

<|endoftext|>


<commit_before>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import shutil
import hashlib
import logging
import threading
import multiprocessing
import time
import datetime
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptography modules
from M2Crypto import RSA

log = logging.getLogger(__name__)

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        while True:
            cur = datetime.datetime.strftime(
                datetime.datetime.now(), '%Y%m%d%H'
            )
            if self.opts['keep_jobs'] == 0:
                return
            jid_root = os.path.join(self.opts['cachedir'], 'jobs')
            for jid in os.listdir(jid_root):
              if int(cur) - int(jid[:10]) > self.opts['keep_jobs']:
                    shutil.rmtree(os.path.join(jid_root, jid))
            time.sleep(60)

    def start(self):
        '''
        Turn on the master server components
        '''
        log.info('Starting the Salt Master')
        multiprocessing.Process(target=self._clear_old_jobs).start()
        reqserv = ReqServer(self.opts)
        reqserv.run()

class Publisher(multiprocessing.Process):
    '''
    The publishing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        multiprocessing.Process.__init__(self)
        self.opts = opts

    def run(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUB)
        pull_sock = context.socket(zmq.PULL)
        pub_uri = 'tcp://%(interface)s:%(publish_port)s' % self.opts
        pull_uri = 'tcp://127.0.0.1:%(publish_pull_port)s' % self.opts
        log.info('Starting the Salt Publisher on %s', pub_uri)
        pub_sock.bind(pub_uri)
        pull_sock.bind(pull_uri)

        while True:
            package = pull_sock.recv()
            log.info('Publishing command')
            pub_sock.send(package)

class ReqServer():
    '''
    Starts up the master request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://%(interface)s:%(ret_port)s' % self.opts
        self.clients = self.context.socket(zmq.XREP)
        self.workers = self.context.socket(zmq.XREQ)
        self.w_uri = 'inproc://workers'
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        log.info('Preparing the root key for local communication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __worker(self, ind):
        '''
        Starts up a worker thread
        '''
        in_socket = self.context.socket(zmq.REP)
        in_socket.connect(self.w_uri)
        m_worker = MWorker(self.opts,
                ind,
                self.master_key,
                self.key,
                self.crypticle)
        work_port = m_worker.port
        m_worker.start()

        out_socket = self.context.socket(zmq.REQ)
        out_socket.connect('tcp://127.0.0.1:%s' % work_port)

        while True:
            package = in_socket.recv()
            out_socket.send(package)
            ret = out_socket.recv()
            in_socket.send(ret)

    def __bind(self):
        '''
        Binds the reply server
        '''
        log.info('Setting up the master communication server')
        self.clients.bind(self.uri)

        self.workers.bind(self.w_uri)

        for ind in range(int(self.opts['worker_threads'])):
            threading.Thread(target=lambda: self.__worker(ind)).start()
            time.sleep(0.1)

        zmq.device(zmq.QUEUE, self.clients, self.workers)

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

class MWorker(multiprocessing.Process):
    '''
    The worker multiprocess instance to manage the backend operations for the
    salt master.
    '''
    def __init__(self, opts, ind, mkey, key, crypticle):
        multiprocessing.Process.__init__(self)
        self.opts = opts
        self.master_key = mkey
        self.key = key
        self.crypticle = crypticle
        self.port = str(ind + int(self.opts['worker_start_port']))

    def __bind(self):
        '''
        Bind to the local port
        '''
        context = zmq.Context(1)
        socket = context.socket(zmq.REP)
        socket.bind('tcp://127.0.0.1:%s' % self.port)

        while True:
            package = socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = datetime.datetime.strftime(
            datetime.datetime.now(), '%Y%m%d%H%M%S%f'
        )
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        log.info('Clear payload received with command %(cmd)s', load)
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        log.info('Pubkey payload received with command %(cmd)s', load)

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        log.info('AES payload received with command %(cmd)s', load)
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are receiving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        log.info('Authentication request from %(id)s', load)
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                log.error(
                    'Authentication attempt from %(id)s failed, the public '
                    'keys did not match. This may be an attempt to compromise '
                    'the Salt cluster.', load
                )
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            log.info('New public key placed in pending for %(id)s', load)
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                log.error(
                    'Authentication attempt from %(id)s failed, the public '
                    'keys in pending did not match. This may be an attempt to '
                    'compromise the Salt cluster.', load
                )
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                log.info(
                    'Authentication failed from host %(id)s, the key is in '
                    'pending and needs to be accepted with saltkey -a %(id)s',
                    load
                )
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        log.info('Authentication accepted from %(id)s', load)
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _find_file(self, path, env='base'):
        '''
        Search the environment for the relative path
        '''
        fnd = {'path': '',
               'rel': ''}
        if not self.opts['file_roots'].has_key(env):
            return fnd
        for root in self.opts['file_roots'][env]:
            full = os.path.join(root, path)
            if os.path.isfile(full):
                fnd['path'] = full
                fnd['rel'] = path
                return fnd
        return fnd

    def _serve_file(self, load):
        '''
        Return a chunk from a file based on the data received
        '''
        ret = {'data': '',
               'dest': ''}
        if not load.has_key('path')\
                or not load.has_key('loc')\
                or not load.has_key('env'):
            return self.crypticle.dumps(ret)
        fnd = self._find_file(load['path'], load['env'])
        if not fnd['path']:
            return self.crypticle.dumps(ret)
        ret['dest'] = fnd['rel']
        fn_ = open(fnd['path'], 'rb')
        fn_.seek(load['loc'])
        ret['data'] = fn_.read(self.opts['file_buffer_size'])
        return self.crypticle.dumps(ret)

    def _file_hash(self, load):
        '''
        Return a file hash, the hash type is set in the master config file
        '''
        if not load.has_key('path')\
                or not load.has_key('env'):
            return False
        path = self._find_file(load['path'], load['env'])
        if not path:
            return self.crypticle.dumps('')
        ret = {}
        ret['hsum'] = getattr(hashlib, self.opts['hash_type'])(
                open(path, 'rb').read()).hexdigest()
        ret['hash_type'] = self.opts['hash_type']
        return self.crypticle.dumps(ret)

    def _master_opts(self, load):
        '''
        Return the master options to the minion
        '''
        return self.crypticle.dumps(self.opts)

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        log.info('Got return from %(id)s for job %(jid)s', load)
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            log.error(
                'An inconsistency occurred, a job was received with a job id '
                'that is not present on the master: %(jid)s', load
            )
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluster data out
        '''
        log.debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        log.debug('Cluster distributed: %s', ret)

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUSH)
        pub_sock.connect('tcp://127.0.0.1:%(publish_pull_port)s' % self.opts)
        pub_sock.send(salt.payload.package(payload))
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start a Master Worker
        '''
        self.__bind()
<commit_msg>Fix issue with path<commit_after>'''
This module contains all fo the routines needed to set up a master server, this
involves preparing the three listeners and the workers needed by the master.
'''
# Import python modules
import os
import shutil
import hashlib
import logging
import threading
import multiprocessing
import time
import datetime
import cPickle as pickle
# Import zeromq
import zmq
# Import salt modules
import salt.utils
import salt.crypt
import salt.payload
import salt.client
# Import cryptography modules
from M2Crypto import RSA

log = logging.getLogger(__name__)

class Master(object):
    '''
    The salt master server
    '''
    def __init__(self, opts):
        '''
        Create a salt master server instance
        '''
        self.opts = opts

    def _clear_old_jobs(self):
        '''
        Clean out the old jobs
        '''
        while True:
            cur = datetime.datetime.strftime(
                datetime.datetime.now(), '%Y%m%d%H'
            )
            if self.opts['keep_jobs'] == 0:
                return
            jid_root = os.path.join(self.opts['cachedir'], 'jobs')
            for jid in os.listdir(jid_root):
              if int(cur) - int(jid[:10]) > self.opts['keep_jobs']:
                    shutil.rmtree(os.path.join(jid_root, jid))
            time.sleep(60)

    def start(self):
        '''
        Turn on the master server components
        '''
        log.info('Starting the Salt Master')
        multiprocessing.Process(target=self._clear_old_jobs).start()
        reqserv = ReqServer(self.opts)
        reqserv.run()

class Publisher(multiprocessing.Process):
    '''
    The publishing interface, a simple zeromq publisher that sends out the
    commands.
    '''
    def __init__(self, opts):
        multiprocessing.Process.__init__(self)
        self.opts = opts

    def run(self):
        '''
        Bind to the interface specified in the configuration file
        '''
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUB)
        pull_sock = context.socket(zmq.PULL)
        pub_uri = 'tcp://%(interface)s:%(publish_port)s' % self.opts
        pull_uri = 'tcp://127.0.0.1:%(publish_pull_port)s' % self.opts
        log.info('Starting the Salt Publisher on %s', pub_uri)
        pub_sock.bind(pub_uri)
        pull_sock.bind(pull_uri)

        while True:
            package = pull_sock.recv()
            log.info('Publishing command')
            pub_sock.send(package)

class ReqServer():
    '''
    Starts up the master request server, minions send results to this
    interface.
    '''
    def __init__(self, opts):
        self.opts = opts
        self.master_key = salt.crypt.MasterKeys(self.opts)
        self.context = zmq.Context(1)
        # Prepare the zeromq sockets
        self.uri = 'tcp://%(interface)s:%(ret_port)s' % self.opts
        self.clients = self.context.socket(zmq.XREP)
        self.workers = self.context.socket(zmq.XREQ)
        self.w_uri = 'inproc://workers'
        # Start the publisher
        self.publisher = Publisher(opts)
        self.publisher.start()
        # Prepare the aes key
        self.key = self.__prep_key()
        self.crypticle = salt.crypt.Crypticle(self.opts['aes'])
        # Make a client
        self.local = salt.client.LocalClient(self.opts['conf_file'])

    def __prep_key(self):
        '''
        A key needs to be placed in the filesystem with permissions 0400 so
        clients are required to run as root.
        '''
        log.info('Preparing the root key for local communication')
        keyfile = os.path.join(self.opts['cachedir'], '.root_key')
        key = salt.crypt.Crypticle.generate_key_string()
        if os.path.isfile(keyfile):
            os.chmod(keyfile, 384)
        open(keyfile, 'w+').write(key)
        os.chmod(keyfile, 256)
        return key

    def __worker(self, ind):
        '''
        Starts up a worker thread
        '''
        in_socket = self.context.socket(zmq.REP)
        in_socket.connect(self.w_uri)
        m_worker = MWorker(self.opts,
                ind,
                self.master_key,
                self.key,
                self.crypticle)
        work_port = m_worker.port
        m_worker.start()

        out_socket = self.context.socket(zmq.REQ)
        out_socket.connect('tcp://127.0.0.1:%s' % work_port)

        while True:
            package = in_socket.recv()
            out_socket.send(package)
            ret = out_socket.recv()
            in_socket.send(ret)

    def __bind(self):
        '''
        Binds the reply server
        '''
        log.info('Setting up the master communication server')
        self.clients.bind(self.uri)

        self.workers.bind(self.w_uri)

        for ind in range(int(self.opts['worker_threads'])):
            threading.Thread(target=lambda: self.__worker(ind)).start()
            time.sleep(0.1)

        zmq.device(zmq.QUEUE, self.clients, self.workers)

    def run(self):
        '''
        Start up the ReqServer
        '''
        self.__bind()

class MWorker(multiprocessing.Process):
    '''
    The worker multiprocess instance to manage the backend operations for the
    salt master.
    '''
    def __init__(self, opts, ind, mkey, key, crypticle):
        multiprocessing.Process.__init__(self)
        self.opts = opts
        self.master_key = mkey
        self.key = key
        self.crypticle = crypticle
        self.port = str(ind + int(self.opts['worker_start_port']))

    def __bind(self):
        '''
        Bind to the local port
        '''
        context = zmq.Context(1)
        socket = context.socket(zmq.REP)
        socket.bind('tcp://127.0.0.1:%s' % self.port)

        while True:
            package = socket.recv()
            payload = salt.payload.unpackage(package)
            ret = salt.payload.package(self._handle_payload(payload))
            socket.send(ret)

    def _prep_jid(self, load):
        '''
        Parses the job return directory, generates a job id and sets up the
        job id directory.
        '''
        jid_root = os.path.join(self.opts['cachedir'], 'jobs')
        jid = datetime.datetime.strftime(
            datetime.datetime.now(), '%Y%m%d%H%M%S%f'
        )
        jid_dir = os.path.join(jid_root, jid)
        if not os.path.isdir(jid_dir):
            os.makedirs(jid_dir)
            pickle.dump(load, open(os.path.join(jid_dir, '.load.p'), 'w+'))
        else:
            return self._prep_jid(load)
        return jid

    def _handle_payload(self, payload):
        '''
        The _handle_payload method is the key method used to figure out what
        needs to be done with communication to the server
        '''
        return {'aes': self._handle_aes,
                'pub': self._handle_pub,
                'clear': self._handle_clear}[payload['enc']](payload['load'])

    def _handle_clear(self, load):
        '''
        Take care of a cleartext command
        '''
        log.info('Clear payload received with command %(cmd)s', load)
        return getattr(self, load['cmd'])(load)

    def _handle_pub(self, load):
        '''
        Handle a command sent via a public key pair
        '''
        log.info('Pubkey payload received with command %(cmd)s', load)

    def _handle_aes(self, load):
        '''
        Handle a command sent via an aes key
        '''
        data = self.crypticle.loads(load)
        log.info('AES payload received with command %(cmd)s', load)
        return getattr(self, data['cmd'])(data)

    def _auth(self, load):
        '''
        Authenticate the client, use the sent public key to encrypt the aes key
        which was generated at start up
        '''
        # 1. Verify that the key we are receiving matches the stored key
        # 2. Store the key if it is not there
        # 3. make an rsa key with the pub key
        # 4. encrypt the aes key as an encrypted pickle
        # 5. package the return and return it
        log.info('Authentication request from %(id)s', load)
        pubfn = os.path.join(self.opts['pki_dir'],
                'minions',
                load['id'])
        pubfn_pend = os.path.join(self.opts['pki_dir'],
                'minions_pre',
                load['id'])
        if self.opts['open_mode']:
            # open mode is turned on, nuts to checks and overwrite whatever
            # is there
            pass
        elif os.path.isfile(pubfn):
            # The key has been accepted check it
            if not open(pubfn, 'r').read() == load['pub']:
                log.error(
                    'Authentication attempt from %(id)s failed, the public '
                    'keys did not match. This may be an attempt to compromise '
                    'the Salt cluster.', load
                )
                ret = {'enc': 'clear',
                       'load': {'ret': False}}
                return ret
        elif not os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This is a new key, stick it in pre
            log.info('New public key placed in pending for %(id)s', load)
            open(pubfn_pend, 'w+').write(load['pub'])
            ret = {'enc': 'clear',
                   'load': {'ret': True}}
            return ret
        elif os.path.isfile(pubfn_pend)\
                and not self.opts['auto_accept']:
            # This key is in pending, if it is the same key ret True, else
            # ret False
            if not open(pubfn_pend, 'r').read() == load['pub']:
                log.error(
                    'Authentication attempt from %(id)s failed, the public '
                    'keys in pending did not match. This may be an attempt to '
                    'compromise the Salt cluster.', load
                )
                return {'enc': 'clear',
                        'load': {'ret': False}}
            else:
                log.info(
                    'Authentication failed from host %(id)s, the key is in '
                    'pending and needs to be accepted with saltkey -a %(id)s',
                    load
                )
                return {'enc': 'clear',
                        'load': {'ret': True}}
        elif not os.path.isfile(pubfn_pend)\
                and self.opts['auto_accept']:
            # This is a new key and auto_accept is turned on
            pass
        else:
            # Something happened that I have not accounted for, FAIL!
            return {'enc': 'clear',
                    'load': {'ret': False}}

        log.info('Authentication accepted from %(id)s', load)
        open(pubfn, 'w+').write(load['pub'])
        key = RSA.load_pub_key(pubfn)
        ret = {'enc': 'pub',
               'pub_key': self.master_key.pub_str,
               'token': self.master_key.token,
               'publish_port': self.opts['publish_port'],
              }
        ret['aes'] = key.public_encrypt(self.opts['aes'], 4)
        if self.opts['cluster_masters']:
            self._send_cluster()
        return ret

    def _find_file(self, path, env='base'):
        '''
        Search the environment for the relative path
        '''
        fnd = {'path': '',
               'rel': ''}
        if not self.opts['file_roots'].has_key(env):
            return fnd
        for root in self.opts['file_roots'][env]:
            full = os.path.join(root, path)
            if os.path.isfile(full):
                fnd['path'] = full
                fnd['rel'] = path
                return fnd
        return fnd

    def _serve_file(self, load):
        '''
        Return a chunk from a file based on the data received
        '''
        ret = {'data': '',
               'dest': ''}
        if not load.has_key('path')\
                or not load.has_key('loc')\
                or not load.has_key('env'):
            return self.crypticle.dumps(ret)
        fnd = self._find_file(load['path'], load['env'])
        if not fnd['path']:
            return self.crypticle.dumps(ret)
        ret['dest'] = fnd['rel']
        fn_ = open(fnd['path'], 'rb')
        fn_.seek(load['loc'])
        ret['data'] = fn_.read(self.opts['file_buffer_size'])
        return self.crypticle.dumps(ret)

    def _file_hash(self, load):
        '''
        Return a file hash, the hash type is set in the master config file
        '''
        if not load.has_key('path')\
                or not load.has_key('env'):
            return False
        path = self._find_file(load['path'], load['env'])['path']
        if not path:
            return self.crypticle.dumps('')
        ret = {}
        ret['hsum'] = getattr(hashlib, self.opts['hash_type'])(
                open(path, 'rb').read()).hexdigest()
        ret['hash_type'] = self.opts['hash_type']
        return self.crypticle.dumps(ret)

    def _master_opts(self, load):
        '''
        Return the master options to the minion
        '''
        return self.crypticle.dumps(self.opts)

    def _return(self, load):
        '''
        Handle the return data sent from the minions
        '''
        # If the return data is invalid, just ignore it
        if not load.has_key('return')\
                or not load.has_key('jid')\
                or not load.has_key('id'):
            return False
        log.info('Got return from %(id)s for job %(jid)s', load)
        jid_dir = os.path.join(self.opts['cachedir'], 'jobs', load['jid'])
        if not os.path.isdir(jid_dir):
            log.error(
                'An inconsistency occurred, a job was received with a job id '
                'that is not present on the master: %(jid)s', load
            )
            return False
        hn_dir = os.path.join(jid_dir, load['id'])
        if not os.path.isdir(hn_dir):
            os.makedirs(hn_dir)
        pickle.dump(load['return'],
                open(os.path.join(hn_dir, 'return.p'), 'w+'))

    def _send_cluster(self):
        '''
        Send the cluster data out
        '''
        log.debug('Sending out cluster data')
        ret = self.local.cmd(self.opts['cluster_masters'],
                'cluster.distrib',
                self._cluster_load(),
                0,
                'list'
                )
        log.debug('Cluster distributed: %s', ret)

    def _cluster_load(self):
        '''
        Generates the data sent to the cluster nodes.
        '''
        minions = {}
        master_pem = ''
        master_conf = open(self.opts['conf_file'], 'r').read()
        minion_dir = os.path.join(self.opts['pki_dir'], 'minions')
        for host in os.listdir(minion_dir):
            pub = os.path.join(minion_dir, host)
            minions[host] = open(pub, 'r').read()
        if self.opts['cluster_mode'] == 'full':
            master_pem = open(os.path.join(self.opts['pki_dir'],
                'master.pem')).read()
        return [minions,
                master_conf,
                master_pem,
                self.opts['conf_file']]

    def publish(self, clear_load):
        '''
        This method sends out publications to the minions
        '''
        if not clear_load.pop('key') == self.key:
            return ''
        jid = self._prep_jid(clear_load)
        payload = {'enc': 'aes'}
        load = {
                'fun': clear_load['fun'],
                'arg': clear_load['arg'],
                'tgt': clear_load['tgt'],
                'jid': jid,
                'ret': clear_load['ret'],
               }
        if clear_load.has_key('tgt_type'):
            load['tgt_type'] = clear_load['tgt_type']
        payload['load'] = self.crypticle.dumps(load)
        context = zmq.Context(1)
        pub_sock = context.socket(zmq.PUSH)
        pub_sock.connect('tcp://127.0.0.1:%(publish_pull_port)s' % self.opts)
        pub_sock.send(salt.payload.package(payload))
        return {'enc': 'clear',
                'load': {'jid': jid}}

    def run(self):
        '''
        Start a Master Worker
        '''
        self.__bind()
<|endoftext|>