<commit_before>import os
import ctypes
import tempfile
import subprocess
import sys
from os.path import isfile

from mock import patch
from six import text_type

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import WindowsAPIError

from pywincffi.kernel32 import file as _file  # used for mocks
from pywincffi.kernel32 import (
    CreateFile, CloseHandle, MoveFileEx, WriteFile, FlushFileBuffers,
    LockFileEx, UnlockFileEx, handle_from_file, ReadFile)

class TestWriteFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.WriteFile`
    """
    def create_handle(self):
        fd, path = tempfile.mkstemp()
        self.addCleanup(os.remove, path)
        file_ = os.fdopen(fd, "w")
        self.addCleanup(file_.close)
        handle = handle_from_file(file_)
        return handle, path

    def test_write_binary(self):
        handle, path = self.create_handle()
        WriteFile(handle, b"hello world")
        FlushFileBuffers(handle)
        with open(path, "rb") as file_:
            self.assertEqual(file_.read(), b"hello world")

    def test_write_binary_num_bytes(self):
        handle, path = self.create_handle()
        WriteFile(handle, b"hello world", nNumberOfBytesToWrite=5)
        FlushFileBuffers(handle)
        with open(path, "rb") as file_:
            self.assertEqual(file_.read(), b"hello")

class TestReadFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.ReadFile`
    """
    def _create_file(self, contents):
        fd, path = tempfile.mkstemp()
        self.addCleanup(os.remove, path)
        with os.fdopen(fd, "wb") as file_:
            file_.write(contents)
        return text_type(path)

    def _handle_to_read_file(self, path):
        _, library = dist.load()
        # OPEN_EXISTING prevents the file from being truncated.
        hFile = CreateFile(
            path,
            dwDesiredAccess=library.GENERIC_READ,
            dwCreationDisposition=library.OPEN_EXISTING,
        )
        self.addCleanup(CloseHandle, hFile)
        return hFile

    def test_write_then_read_bytes_ascii(self):
        path = self._create_file(b"test_write_then_read_bytes_ascii")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 1024)
        self.assertEqual(contents, b"test_write_then_read_bytes_ascii")

    def test_write_then_read_null_bytes(self):
        path = self._create_file(b"hello\x00world")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 1024)
        self.assertEqual(contents, b"hello\x00world")

    def test_write_then_read_partial(self):
        path = self._create_file(b"test_write_then_read_partial")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 4)
        self.assertEqual(contents, b"test")

class TestMoveFileEx(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.MoveFileEx`
    """
    def test_replaces_file(self):
        # Destination file exists, this should replace it.
        file_contents = self.random_string(12)
        fd, path1 = tempfile.mkstemp()

        with os.fdopen(fd, "w") as file_:
            file_.write(file_contents)

        fd, path2 = tempfile.mkstemp()
        self.addCleanup(os.remove, path2)
        os.close(fd)

        path1 = text_type(path1)  # pylint: disable=redefined-variable-type
        path2 = text_type(path2)  # pylint: disable=redefined-variable-type
        MoveFileEx(path1, path2)

        with open(path2, "r") as file_:
            self.assertEqual(file_.read(), file_contents)

        self.assertFalse(isfile(path1))

    def test_renames_file(self):
        # Destination file does not exist, this should create it.
        file_contents = self.random_string(12)
        fd, path1 = tempfile.mkstemp()
        path2 = path1 + ".new"

        with os.fdopen(fd, "w") as file_:
            file_.write(file_contents)

        path1 = text_type(path1)  # pylint: disable=redefined-variable-type
        path2 = text_type(path2)  # pylint: disable=redefined-variable-type
        MoveFileEx(path1, path2)

        with open(path2, "r") as file_:
            self.assertEqual(file_.read(), file_contents)

        self.assertFalse(isfile(path1))

    def test_run_delete_after_reboot(self):
        fd, path = tempfile.mkstemp()
        os.close(fd)

        path = text_type(path)  # pylint: disable=redefined-variable-type

        _, library = dist.load()
        try:
            MoveFileEx(path, None, dwFlags=library.MOVEFILE_DELAY_UNTIL_REBOOT)
        except WindowsAPIError as error:
            # If we're not an administrator then we don't
            # have the permissions to perform this kind of
            # action.
            if error.errno == library.ERROR_ACCESS_DENIED:
                self.assertFalse(ctypes.windll.shell32.IsUserAnAdmin())
                return

            raise
        else:
            self.assertTrue(isfile(path))

class TestCreateFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.CreateFile`
    """
    def test_creates_file(self):
        fd, path = tempfile.mkstemp()
        os.close(fd)
        os.remove(path)

        path = text_type(path)  # pylint: disable=redefined-variable-type
        handle = CreateFile(path, 0)
        self.addCleanup(CloseHandle, handle)
        self.assertTrue(isfile(path))

    def test_default_create_disposition(self):
        # The default creation disposition should
        # overwrite an existing file.
        fd, path = tempfile.mkstemp()
        with os.fdopen(fd, "w") as file_:
            file_.write("Hello, world.")
            file_.flush()
            os.fsync(file_.fileno())

        path = text_type(path)  # pylint: disable=redefined-variable-type
        handle = CreateFile(path, 0)
        self.addCleanup(CloseHandle, handle)

        with open(path, "r") as file_:
            self.assertEqual(file_.read(), "")

    def test_ignores_error_already_existed(self):
        _, library = dist.load()
        fd, path = tempfile.mkstemp()
        os.close(fd)
        os.remove(path)

        def raise_(*_):
            raise WindowsAPIError("", "", library.ERROR_ALREADY_EXISTS)

        with patch.object(_file, "error_check", side_effect=raise_):
            path = text_type(path)  # pylint: disable=redefined-variable-type
            handle = CreateFile(
                path, 0, dwCreationDisposition=library.CREATE_ALWAYS)
            self.addCleanup(CloseHandle, handle)

        # If we've made it this far, the exception was ignored by CreateFile

    def test_raises_other_errors_for_create_always(self):
        _, library = dist.load()

        with self.assertRaises(WindowsAPIError) as error:
            handle = CreateFile(
                u"", 0, dwCreationDisposition=library.CREATE_ALWAYS)
            self.addCleanup(CloseHandle, handle)

        self.assertEqual(error.exception.errno, library.ERROR_PATH_NOT_FOUND)

class LockFileCase(TestCase):
    def setUp(self):
        super(LockFileCase, self).setUp()
        fd, path = tempfile.mkstemp()
        self.path = path
        os.close(fd)
        self.addCleanup(os.remove, path)
        _, library = dist.load()
        path = text_type(path)  # pylint: disable=redefined-variable-type
        self.handle = CreateFile(path, library.GENERIC_WRITE)
        self.addCleanup(CloseHandle, self.handle)

        WriteFile(self.handle, b"hello")

class TestLockFileEx(LockFileCase):
    """
    Tests for :func:`pywincffi.kernel32.LockFileEx`
    """
    def test_lock_causes_subprocess_read_failure(self):
        _, library = dist.load()
        LockFileEx(
            self.handle,
            library.LOCKFILE_EXCLUSIVE_LOCK |
            library.LOCKFILE_FAIL_IMMEDIATELY,
            0, 1024)

        with self.assertRaises(subprocess.CalledProcessError):
            subprocess.check_call(
                [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
                stderr=subprocess.PIPE
            )

    def test_no_lock_allows_subprocess_read_python3(self):
        # This makes sure that if the default behavior changes or varies
        # between Python versions we catch it.  Without this there's not a way
        # to ensure that test_lock_causes_subprocess_read_failure() is really
        # testing the behavior of LockFileEx()
        subprocess.check_call(
            [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
            stderr=subprocess.PIPE
        )

class TestUnlockFileEx(LockFileCase):
    """
    Tests for :func:`pywincffi.kernel32.UnlockFileEx`
    """
    def test_unlock_file(self):
        _, library = dist.load()
        LockFileEx(
            self.handle,
            library.LOCKFILE_EXCLUSIVE_LOCK |
            library.LOCKFILE_FAIL_IMMEDIATELY,
            0, 1024)

        with self.assertRaises(subprocess.CalledProcessError):
            subprocess.check_call(
                [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
                stderr=subprocess.PIPE
            )

        UnlockFileEx(self.handle, 0, 1024)

        subprocess.check_call([
            sys.executable, "-c", "open(%r, 'r').read()" % self.path])
<commit_msg>Tests - Ensure file tests cleanup after themselves<commit_after>import os
import ctypes
import tempfile
import subprocess
import sys
from os.path import isfile

from mock import patch
from six import text_type

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import WindowsAPIError

from pywincffi.kernel32 import file as _file  # used for mocks
from pywincffi.kernel32 import (
    CreateFile, CloseHandle, MoveFileEx, WriteFile, FlushFileBuffers,
    LockFileEx, UnlockFileEx, handle_from_file, ReadFile)

class TestWriteFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.WriteFile`
    """
    def create_handle(self):
        fd, path = tempfile.mkstemp()
        self.addCleanup(os.remove, path)
        file_ = os.fdopen(fd, "w")
        self.addCleanup(file_.close)
        handle = handle_from_file(file_)
        return handle, path

    def test_write_binary(self):
        handle, path = self.create_handle()
        WriteFile(handle, b"hello world")
        FlushFileBuffers(handle)
        with open(path, "rb") as file_:
            self.assertEqual(file_.read(), b"hello world")

    def test_write_binary_num_bytes(self):
        handle, path = self.create_handle()
        WriteFile(handle, b"hello world", nNumberOfBytesToWrite=5)
        FlushFileBuffers(handle)
        with open(path, "rb") as file_:
            self.assertEqual(file_.read(), b"hello")

class TestReadFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.ReadFile`
    """
    def _create_file(self, contents):
        fd, path = tempfile.mkstemp()
        self.addCleanup(os.remove, path)
        with os.fdopen(fd, "wb") as file_:
            file_.write(contents)
        return text_type(path)

    def _handle_to_read_file(self, path):
        _, library = dist.load()
        # OPEN_EXISTING prevents the file from being truncated.
        hFile = CreateFile(
            path,
            dwDesiredAccess=library.GENERIC_READ,
            dwCreationDisposition=library.OPEN_EXISTING,
        )
        self.addCleanup(CloseHandle, hFile)
        return hFile

    def test_write_then_read_bytes_ascii(self):
        path = self._create_file(b"test_write_then_read_bytes_ascii")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 1024)
        self.assertEqual(contents, b"test_write_then_read_bytes_ascii")

    def test_write_then_read_null_bytes(self):
        path = self._create_file(b"hello\x00world")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 1024)
        self.assertEqual(contents, b"hello\x00world")

    def test_write_then_read_partial(self):
        path = self._create_file(b"test_write_then_read_partial")
        hFile = self._handle_to_read_file(path)
        contents = ReadFile(hFile, 4)
        self.assertEqual(contents, b"test")

class TestMoveFileEx(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.MoveFileEx`
    """
    def test_replaces_file(self):
        # Destination file exists, this should replace it.
        file_contents = self.random_string(12)
        fd, path1 = tempfile.mkstemp()

        with os.fdopen(fd, "w") as file_:
            file_.write(file_contents)

        fd, path2 = tempfile.mkstemp()
        self.addCleanup(os.remove, path2)
        os.close(fd)

        path1 = text_type(path1)  # pylint: disable=redefined-variable-type
        path2 = text_type(path2)  # pylint: disable=redefined-variable-type
        MoveFileEx(path1, path2)

        with open(path2, "r") as file_:
            self.assertEqual(file_.read(), file_contents)

        self.assertFalse(isfile(path1))

    def _remove_file(self, path):
        try:
            os.remove(path)
        except OSError:
            pass

    def test_renames_file(self):
        # Destination file does not exist, this should create it.
        file_contents = self.random_string(12)
        fd, path1 = tempfile.mkstemp()
        path2 = path1 + ".new"
        self.addCleanup(self._remove_file, path1)
        self.addCleanup(self._remove_file, path2)

        with os.fdopen(fd, "w") as file_:
            file_.write(file_contents)

        path1 = text_type(path1)  # pylint: disable=redefined-variable-type
        path2 = text_type(path2)  # pylint: disable=redefined-variable-type
        MoveFileEx(path1, path2)

        with open(path2, "r") as file_:
            self.assertEqual(file_.read(), file_contents)

        self.assertFalse(isfile(path1))

    def test_run_delete_after_reboot(self):
        fd, path = tempfile.mkstemp('-removed-on-next-reboot')
        os.close(fd)

        path = text_type(path)  # pylint: disable=redefined-variable-type

        _, library = dist.load()
        try:
            MoveFileEx(path, None, dwFlags=library.MOVEFILE_DELAY_UNTIL_REBOOT)
        except WindowsAPIError as error:
            # If we're not an administrator then we don't
            # have the permissions to perform this kind of
            # action.
            if error.errno == library.ERROR_ACCESS_DENIED:
                self.addCleanup(os.remove, path)
                self.assertFalse(ctypes.windll.shell32.IsUserAnAdmin())
                return

            raise
        else:
            self.assertTrue(isfile(path))

class TestCreateFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.CreateFile`
    """
    def test_creates_file(self):
        fd, path = tempfile.mkstemp()
        os.close(fd)
        os.remove(path)

        path = text_type(path)  # pylint: disable=redefined-variable-type
        handle = CreateFile(path, 0)
        self.addCleanup(os.remove, path)
        self.addCleanup(CloseHandle, handle)
        self.assertTrue(isfile(path))

    def test_default_create_disposition(self):
        # The default creation disposition should
        # overwrite an existing file.
        fd, path = tempfile.mkstemp()
        with os.fdopen(fd, "w") as file_:
            file_.write("Hello, world.")
            file_.flush()
            os.fsync(file_.fileno())

        path = text_type(path)  # pylint: disable=redefined-variable-type
        handle = CreateFile(path, 0)
        self.addCleanup(os.remove, path)
        self.addCleanup(CloseHandle, handle)

        with open(path, "r") as file_:
            self.assertEqual(file_.read(), "")

    def test_ignores_error_already_existed(self):
        _, library = dist.load()
        fd, path = tempfile.mkstemp()
        os.close(fd)
        os.remove(path)

        def raise_(*_):
            raise WindowsAPIError("", "", library.ERROR_ALREADY_EXISTS)

        with patch.object(_file, "error_check", side_effect=raise_):
            path = text_type(path)  # pylint: disable=redefined-variable-type
            handle = CreateFile(
                path, 0, dwCreationDisposition=library.CREATE_ALWAYS)
            self.addCleanup(os.remove, path)
            self.addCleanup(CloseHandle, handle)

        # If we've made it this far, the exception was ignored by CreateFile

    def test_raises_other_errors_for_create_always(self):
        _, library = dist.load()

        with self.assertRaises(WindowsAPIError) as error:
            handle = CreateFile(
                u"", 0, dwCreationDisposition=library.CREATE_ALWAYS)
            self.addCleanup(CloseHandle, handle)

        self.assertEqual(error.exception.errno, library.ERROR_PATH_NOT_FOUND)

class LockFileCase(TestCase):
    def setUp(self):
        super(LockFileCase, self).setUp()
        fd, path = tempfile.mkstemp()
        self.path = path
        os.close(fd)
        self.addCleanup(os.remove, path)
        _, library = dist.load()
        path = text_type(path)  # pylint: disable=redefined-variable-type
        self.handle = CreateFile(path, library.GENERIC_WRITE)
        self.addCleanup(CloseHandle, self.handle)

        WriteFile(self.handle, b"hello")

class TestLockFileEx(LockFileCase):
    """
    Tests for :func:`pywincffi.kernel32.LockFileEx`
    """
    def test_lock_causes_subprocess_read_failure(self):
        _, library = dist.load()
        LockFileEx(
            self.handle,
            library.LOCKFILE_EXCLUSIVE_LOCK |
            library.LOCKFILE_FAIL_IMMEDIATELY,
            0, 1024)

        with self.assertRaises(subprocess.CalledProcessError):
            subprocess.check_call(
                [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
                stderr=subprocess.PIPE
            )

    def test_no_lock_allows_subprocess_read_python3(self):
        # This makes sure that if the default behavior changes or varies
        # between Python versions we catch it.  Without this there's not a way
        # to ensure that test_lock_causes_subprocess_read_failure() is really
        # testing the behavior of LockFileEx()
        subprocess.check_call(
            [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
            stderr=subprocess.PIPE
        )

class TestUnlockFileEx(LockFileCase):
    """
    Tests for :func:`pywincffi.kernel32.UnlockFileEx`
    """
    def test_unlock_file(self):
        _, library = dist.load()
        LockFileEx(
            self.handle,
            library.LOCKFILE_EXCLUSIVE_LOCK |
            library.LOCKFILE_FAIL_IMMEDIATELY,
            0, 1024)

        with self.assertRaises(subprocess.CalledProcessError):
            subprocess.check_call(
                [sys.executable, "-c", "open(%r, 'r').read()" % self.path],
                stderr=subprocess.PIPE
            )

        UnlockFileEx(self.handle, 0, 1024)

        subprocess.check_call([
            sys.executable, "-c", "open(%r, 'r').read()" % self.path])
<|endoftext|>


<commit_before>"""Mealboard recipe exporter.

This script goes to the Mealboard site and downloads all of your recipes
and puts them into a standarized format. See README.md for more details.
"""
# The straightforward imports:
import argparse
import os
import re
import sys

if sys.version_info < (3, 4):
    print('WARNING: This script is only tested to work with '
          'Python 3.4 and above.\nYou seem to be running version {}.{}.\n'
          'If you run into errors, please re-run with a higher version of '
          'Python.'.format(sys.version_info.major, sys.version_info.minor))

# The imports that may not yet be installed:
try:
    from lxml.html import fromstring
except ImportError:
    print('lxml python module must be installed for this script to work.')
    print('Install this module and re-run this script.')
    print('Typically something like "pip install lxml" should work.')
    quit()
try:
    import requests
    from requests.adapters import HTTPAdapter
    from requests.packages.urllib3.exceptions import InsecureRequestWarning
except ImportError:
    print('requests python module must be installed for this script to work.')
    print('Install this module and re-run this script.')
    print('Typically something like "pip install requests" should work.')
    quit()

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

mealboard_url = 'http://mealboard.macminicolo.net/mealboard/'

def login(url, loginData):
    """Login into mealboard and create session.

    Store the session cookie so we can use it for getting the recipes.
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0Win64; x64) \
        AppleWebKit/537.36 (KHTML, like Gecko)\
         Chrome/58.0.3029.110 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,\
        image/webp,*/*;q=0.8',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Referer': '{}login.jsp'.format(mealboard_url),
    }

    s = requests.Session()
    s.mount('http://', HTTPAdapter(max_retries=3))
    s.mount('https://', HTTPAdapter(max_retries=3))

    try:
        r = s.post(url,
                   headers=headers,
                   timeout=30,
                   verify=False,
                   data=loginData,
                   allow_redirects=True
                   )

        if r.status_code == 200:
            content = (r.content).decode('utf-8')
            return content, s
        else:
            return None, None

    except Exception as E:
        print(E)
        return None, None

def getHtml(url, session):
    """Use a simple Get requests to get all html pages."""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
        'AppleWebKit/537.36 (KHTML, like Gecko) '
        'Chrome/58.0.3029.110 '
        'Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,'
        'image/webp,*/*;q=0.8',
        'Referer': '{}recipeedit.jsp'.format(mealboard_url)
    }

    try:
        r = session.get(url,
                        headers=headers,
                        timeout=30,
                        verify=False,
                        allow_redirects=True
                        )

        if r.status_code == 200:
            content = (r.content).decode('utf-8')
            return content, session
        else:
            return None, None

    except Exception as E:
        print(E)
        return None, None

def parseRecipeEditPage(content, recipeData):
    """Parse one recipe.

    Parse and extract Category, Makes, Prep Time, Cook Time from Edit Page
    """
    html = fromstring(content)

    # Get categories
    try:
        categories = list()

        categories = html.xpath(
            '//select[@name="category"]/option[@selected="selected"]/@value'
        )

        recipeData.update({'Category': ", ".join(categories)})
    except Exception as E:
        print(E)
        pass

    # Servings (Makes)
    try:
        servings = html.xpath('//input[@name="num_servings"]/@value')[0]
        recipeData.update({'Servings': servings})
    except BaseException:
        pass

    # Prep Time
    try:
        PrepTimeContent = 'PT'
        PrepTime = ''

        prepHour = html.xpath('//input[@name="prep_time_hr"]/@value')[0]
        if len(prepHour.strip()) > 0:
            PrepTimeContent = "{0}{1}H".format(PrepTimeContent, prepHour)
            PrepTime = "{0} H".format(prepHour)

        prepMin = html.xpath('//input[@name="prep_time_min"]/@value')[0]
        if len(prepMin.strip()) > 0:
            PrepTimeContent = "{0}{1}M".format(PrepTimeContent, prepMin)
            PrepTime = "{0} {1} Min".format(PrepTime, prepMin)

        if len(PrepTimeContent) > 2:
            recipeData.update({'PrepTimeContent': PrepTimeContent})
            recipeData.update({'PrepTime': PrepTime.strip()})
    except BaseException:
        pass

    # Cook Time
    try:
        CookTimeContent = 'PT'
        CookTime = ''

        cookHour = html.xpath('//input[@name="cooking_time_hr"]/@value')[0]
        if len(cookHour.strip()) > 0:
            CookTimeContent = "{0}{1}H".format(CookTimeContent, cookHour)
            CookTime = "{0} H".format(cookHour)

        cookMin = html.xpath('//input[@name="cooking_time_min"]/@value')[0]
        if len(cookMin.strip()) > 0:
            CookTimeContent = "{0}{1}M".format(CookTimeContent, cookMin)
            CookTime = "{0} {1} Min".format(CookTime, cookMin)

        if len(CookTimeContent) > 2:
            recipeData.update({'CookTimeContent': CookTimeContent})
            recipeData.update({'CookTime': CookTime.strip()})
    except BaseException:
        pass

    return recipeData

def parseRecipeHtml(content, recipeData):
    """Parse one recipe.

    Parse missing informations and convert recipe html code
    into new format. using http://schema.org/Recipe
    """
    html = fromstring(content)

    # Load images and generate html code
    try:
        recipeImages = list()

        images = html.xpath('//div[@class="photo"]/img/@src')
        for image in images:
            img_search = ""
            image = img_search.format(image, recipeData['Name'])
            recipeImages.append(image)

        recipeData.update({'Images': "\n".join(recipeImages)})
    except Exception as E:
        print(E)
        pass

    # get recipe description
    try:
        recipeDescription = html.xpath('//div[@class="description"]/text()')[0]
        recipeData.update({'Description': recipeDescription})
    except Exception as E:
        # print(E)
        pass

    # Load ingredients and generate html code
    try:
        recipeIngredients = list()

        ingredients = html.xpath('//div[@class="ingredientsBox"]/text()')
        for ingredient in ingredients:
            if len(ingredient.strip()) > 0:
                ingredient = "{0}
".format(
                    ingredient.strip())
                recipeIngredients.append(ingredient)

        recipeData.update({'Ingredients': "\n".join(recipeIngredients)})
    except Exception as E:
        # print(E)
        pass

    # Load instructions and generate html code
    try:
        recipeInstructions = list()

        instructions = html.xpath('//div[@class="preparationBox"]/text()')
        for instruction in instructions:
            if len(instruction.strip()) > 0:
                instruction = "{0}
\n".format(
                    instruction.strip())
                recipeInstructions.append(instruction)

        recipeData.update({'Instructions': "\n".join(recipeInstructions)})
    except Exception as E:
        # print(E)
        pass

    # New html5 template
    recipeHtml = """
        <!doctype html>

                <title>{0}</title>

\n

                        {0}

\n
                Category :
                    {1}\n

\n
                {2}

\n
                Description:
\n
                {3}
\n

\n
                Prep Time:
                <meta itemprop="prepTime" content="{4}">{5}
\n
                Cook time:
                <meta itemprop="cookTime" content="{6}">{7}
\n
                Servings:
                {8}
\n

\n
                Ingredients:
\n
                {9}

\n
                Instructions:
\n
                {10}

        """.format(
        recipeData['Name'],
        recipeData['Category'],
        recipeData['Images'],
        recipeData['Description'],
        recipeData['PrepTimeContent'],
        recipeData['PrepTime'],
        recipeData['CookTimeContent'],
        recipeData['CookTime'],
        recipeData['Servings'],
        recipeData['Ingredients'],
        recipeData['Instructions'],
        recipeData['sourceUrl']
    )

    return recipeHtml

def main(user, pin, location):
    """Do the actual work.

    This is the primary work funtion that pulls the recipes from the site
    and re-formats them, then saves the results.
    """
    loginData = {
        'username': user,
        'pin': pin
    }

    # Login
    content, session = login(
        '{}loginsubmit.jsp'.format(mealboard_url),
        loginData)

    if content is None:
        print('Login Failed')
    else:
        if 'name="recipe_detail_frame"' in content:
            print('login OK')

            # Get recipes list
            content, session = getHtml('{}recipelist.jsp?RECIPE_NAME=NEW%20RECIPE'.format(mealboard_url), session)

            if content is not None:
                if 'recipelist.css' in content:

                    html = fromstring(content)
                    if html is not None:

                        # Load all recipe names in the left menu
                        recipes = html.xpath('//table/tr/@onclick')
                        for recipe in recipes[2:]:

                            recipeData = {
                                'Name': '',
                                'Images': '',
                                'Servings': '',
                                'PrepTime': '',
                                'PrepTimeContent': '',
                                'CookTime': '',
                                'CookTimeContent': '',
                                'Category': '',
                                'Description': '',
                                'Ingredients': '',
                                'Instructions': '',
                                'sourceUrl': '',
                                'fileName': ''
                            }

                            if "loadRecipeDetail" in recipe:
                                recipeName = recipe.split(
                                    "loadRecipeDetail(\'")[1]
                                recipeName = recipeName.split('\')')[0].strip()

                                recipeData.update({'Name': recipeName})

                                print(recipeName.encode())

                                fileName = "".join(
                                    [c for c in recipeName if re.match(r'\w', c)])
                                recipeData.update({'fileName': fileName})

                                # Load "Edit Page" to get propper values for
                                # Category / Makes / PrepTime / CookTime
                                recipeEditUrl = "{}recipeform.jsp?RECIPE_NAME={}".format(
                                    mealboard_url,
                                    recipeData['Name'])
                                content, session = getHtml(
                                    recipeEditUrl,
                                    session
                                )
                                if content is not None:
                                    recipeData = parseRecipeEditPage(
                                        content,
                                        recipeData
                                    )

                                # Download html page for recipe
                                recipeHtmlUrl = "{}recipeview.jsp?RECIPE_NAME={}".format(
                                    mealboard_url,
                                    recipeData['Name'])
                                recipeData.update({'sourceUrl': recipeHtmlUrl})

                                content, session = getHtml(
                                    recipeHtmlUrl, session)
                                if content is not None:

                                    # If we want to save the original, HTML recipe
                                    # TODO(Michael): Maybe make this a flag/option?
                                    if False:
                                        fileName = "./html/{0}.html".format(
                                            recipeData['fileName'])
                                        with open(fileName, 'w', encoding="utf-8") as fp:
                                            fp.write(content)

                                    # parse and convert recipe html page
                                    recipeHtml = parseRecipeHtml(
                                        content, recipeData)

                                    # Save new html page
                                    if not os.path.exists(location):
                                        os.makedirs(location)
                                    fileName = "{}/{}.html".format(
                                        location,
                                        recipeData['fileName']
                                    )
                                    with open(fileName, 'w', encoding='utf-8') as fp:
                                        fp.write(recipeHtml)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    user_default = 'MEALBOARD_USER env variable'
    pin_default = 'MEALBOARD_PIN env variable'

    parser.add_argument('-u',
                        action='store',
                        dest='user',
                        default=user_default,
                        help='Mealboard user to use to connect')
    parser.add_argument('-p',
                        action='store',
                        dest='pin',
                        default=pin_default,
                        help='Mealboard PIN to use to connect')
    parser.add_argument('-l',
                        action='store',
                        dest='location',
                        default='./recipes',
                        help='Location to store recipes')
    results = parser.parse_args()

    if (results.user == user_default):
        if ('MEALBOARD_USER' in os.environ):
            results.user = os.environ['MEALBOARD_USER']
        else:
            print('Mealboard user not set. Use the -u flag to set this, or '
                  'set the\nMEALBOARD_USER environment variable.')
            quit()
    if (results.pin == pin_default):
        if ('MEALBOARD_PIN' in os.environ):
            results.pin = os.environ['MEALBOARD_PIN']
        else:
            print('Mealboard pin not set. Use the -p flag to set this, or '
                  'set the\nMEALBOARD_PIN environment variable.')
            quit()

    main(results.user, results.pin, results.location)
<commit_msg>Initial class object.<commit_after>"""Mealboard recipe exporter.

This script goes to the Mealboard site and downloads all of your recipes
and puts them into a standarized format. See README.md for more details.
"""
# The straightforward imports:
import argparse
import os
import re
import sys

if sys.version_info < (3, 4):
    print('WARNING: This script is only tested to work with '
          'Python 3.4 and above.\nYou seem to be running version {}.{}.\n'
          'If you run into errors, please re-run with a higher version of '
          'Python.'.format(sys.version_info.major, sys.version_info.minor))

# The imports that may not yet be installed:
try:
    from lxml.html import fromstring
except ImportError:
    print('lxml python module must be installed for this script to work.')
    print('Install this module and re-run this script.')
    print('Typically something like "pip install lxml" should work.')
    quit()
try:
    import requests
    from requests.adapters import HTTPAdapter
    from requests.packages.urllib3.exceptions import InsecureRequestWarning
except ImportError:
    print('requests python module must be installed for this script to work.')
    print('Install this module and re-run this script.')
    print('Typically something like "pip install requests" should work.')
    quit()

requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

mealboard_url = 'http://mealboard.macminicolo.net/mealboard/'

class Recipe(object):
    """One recipe."""

    def __init__(self, recipe_name):
        """Initialize the recipe."""
        self.html = ''
        self.name = recipe_name

    def load(self):
        """Load the recipe from Mealboard's site."""
        print('Loading: {}'.format(self.name))

def login(url, loginData):
    """Login into mealboard and create session.

    Store the session cookie so we can use it for getting the recipes.
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0Win64; x64) \
        AppleWebKit/537.36 (KHTML, like Gecko)\
         Chrome/58.0.3029.110 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,\
        image/webp,*/*;q=0.8',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Referer': '{}login.jsp'.format(mealboard_url),
    }

    s = requests.Session()
    s.mount('http://', HTTPAdapter(max_retries=3))
    s.mount('https://', HTTPAdapter(max_retries=3))

    try:
        r = s.post(url,
                   headers=headers,
                   timeout=30,
                   verify=False,
                   data=loginData,
                   allow_redirects=True
                   )

        if r.status_code == 200:
            content = (r.content).decode('utf-8')
            return content, s
        else:
            return None, None

    except Exception as E:
        print(E)
        return None, None

def getHtml(url, session):
    """Use a simple Get requests to get all html pages."""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
        'AppleWebKit/537.36 (KHTML, like Gecko) '
        'Chrome/58.0.3029.110 '
        'Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,'
        'image/webp,*/*;q=0.8',
        'Referer': '{}recipeedit.jsp'.format(mealboard_url)
    }

    try:
        r = session.get(url,
                        headers=headers,
                        timeout=30,
                        verify=False,
                        allow_redirects=True
                        )

        if r.status_code == 200:
            content = (r.content).decode('utf-8')
            return content, session
        else:
            return None, None

    except Exception as E:
        print(E)
        return None, None

def parseRecipeEditPage(content, recipeData):
    """Parse one recipe.

    Parse and extract Category, Makes, Prep Time, Cook Time from Edit Page
    """
    html = fromstring(content)

    # Get categories
    try:
        categories = list()

        categories = html.xpath(
            '//select[@name="category"]/option[@selected="selected"]/@value'
        )

        recipeData.update({'Category': ", ".join(categories)})
    except Exception as E:
        print(E)
        pass

    # Servings (Makes)
    try:
        servings = html.xpath('//input[@name="num_servings"]/@value')[0]
        recipeData.update({'Servings': servings})
    except BaseException:
        pass

    # Prep Time
    try:
        PrepTimeContent = 'PT'
        PrepTime = ''

        prepHour = html.xpath('//input[@name="prep_time_hr"]/@value')[0]
        if len(prepHour.strip()) > 0:
            PrepTimeContent = "{0}{1}H".format(PrepTimeContent, prepHour)
            PrepTime = "{0} H".format(prepHour)

        prepMin = html.xpath('//input[@name="prep_time_min"]/@value')[0]
        if len(prepMin.strip()) > 0:
            PrepTimeContent = "{0}{1}M".format(PrepTimeContent, prepMin)
            PrepTime = "{0} {1} Min".format(PrepTime, prepMin)

        if len(PrepTimeContent) > 2:
            recipeData.update({'PrepTimeContent': PrepTimeContent})
            recipeData.update({'PrepTime': PrepTime.strip()})
    except BaseException:
        pass

    # Cook Time
    try:
        CookTimeContent = 'PT'
        CookTime = ''

        cookHour = html.xpath('//input[@name="cooking_time_hr"]/@value')[0]
        if len(cookHour.strip()) > 0:
            CookTimeContent = "{0}{1}H".format(CookTimeContent, cookHour)
            CookTime = "{0} H".format(cookHour)

        cookMin = html.xpath('//input[@name="cooking_time_min"]/@value')[0]
        if len(cookMin.strip()) > 0:
            CookTimeContent = "{0}{1}M".format(CookTimeContent, cookMin)
            CookTime = "{0} {1} Min".format(CookTime, cookMin)

        if len(CookTimeContent) > 2:
            recipeData.update({'CookTimeContent': CookTimeContent})
            recipeData.update({'CookTime': CookTime.strip()})
    except BaseException:
        pass

    return recipeData

def parseRecipeHtml(content, recipeData):
    """Parse one recipe.

    Parse missing informations and convert recipe html code
    into new format. using http://schema.org/Recipe
    """
    html = fromstring(content)

    # Load images and generate html code
    try:
        recipeImages = list()

        images = html.xpath('//div[@class="photo"]/img/@src')
        for image in images:
            img_search = ""
            image = img_search.format(image, recipeData['Name'])
            recipeImages.append(image)

        recipeData.update({'Images': "\n".join(recipeImages)})
    except Exception as E:
        print(E)
        pass

    # get recipe description
    try:
        recipeDescription = html.xpath('//div[@class="description"]/text()')[0]
        recipeData.update({'Description': recipeDescription})
    except Exception as E:
        # print(E)
        pass

    # Load ingredients and generate html code
    try:
        recipeIngredients = list()

        ingredients = html.xpath('//div[@class="ingredientsBox"]/text()')
        for ingredient in ingredients:
            if len(ingredient.strip()) > 0:
                ingredient = "{0}
".format(
                    ingredient.strip())
                recipeIngredients.append(ingredient)

        recipeData.update({'Ingredients': "\n".join(recipeIngredients)})
    except Exception as E:
        # print(E)
        pass

    # Load instructions and generate html code
    try:
        recipeInstructions = list()

        instructions = html.xpath('//div[@class="preparationBox"]/text()')
        for instruction in instructions:
            if len(instruction.strip()) > 0:
                instruction = "{0}
\n".format(
                    instruction.strip())
                recipeInstructions.append(instruction)

        recipeData.update({'Instructions': "\n".join(recipeInstructions)})
    except Exception as E:
        # print(E)
        pass

    # New html5 template
    recipeHtml = """
        <!doctype html>

                <title>{0}</title>

\n

                        {0}

\n
                Category :
                    {1}\n

\n
                {2}

\n
                Description:
\n
                {3}
\n

\n
                Prep Time:
                <meta itemprop="prepTime" content="{4}">{5}
\n
                Cook time:
                <meta itemprop="cookTime" content="{6}">{7}
\n
                Servings:
                {8}
\n

\n
                Ingredients:
\n
                {9}

\n
                Instructions:
\n
                {10}

        """.format(
        recipeData['Name'],
        recipeData['Category'],
        recipeData['Images'],
        recipeData['Description'],
        recipeData['PrepTimeContent'],
        recipeData['PrepTime'],
        recipeData['CookTimeContent'],
        recipeData['CookTime'],
        recipeData['Servings'],
        recipeData['Ingredients'],
        recipeData['Instructions'],
        recipeData['sourceUrl']
    )

    return recipeHtml

def main(user, pin, location):
    """Do the actual work.

    This is the primary work funtion that pulls the recipes from the site
    and re-formats them, then saves the results.
    """
    loginData = {
        'username': user,
        'pin': pin
    }

    # Login
    content, session = login(
        '{}loginsubmit.jsp'.format(mealboard_url),
        loginData)

    if content is None:
        print('Login Failed')
    else:
        if 'name="recipe_detail_frame"' in content:
            print('login OK')

            # Get recipes list
            content, session = getHtml('{}recipelist.jsp?RECIPE_NAME=NEW%20RECIPE'.format(mealboard_url), session)

            if content is not None:
                if 'recipelist.css' in content:

                    html = fromstring(content)
                    if html is not None:

                        # Load all recipe names in the left menu
                        recipes = html.xpath('//table/tr/@onclick')
                        for recipe in recipes[2:]:

                            recipeData = {
                                'Name': '',
                                'Images': '',
                                'Servings': '',
                                'PrepTime': '',
                                'PrepTimeContent': '',
                                'CookTime': '',
                                'CookTimeContent': '',
                                'Category': '',
                                'Description': '',
                                'Ingredients': '',
                                'Instructions': '',
                                'sourceUrl': '',
                                'fileName': ''
                            }

                            if "loadRecipeDetail" in recipe:
                                recipeName = recipe.split(
                                    "loadRecipeDetail(\'")[1]
                                recipeName = recipeName.split('\')')[0].strip()

                                recipeData.update({'Name': recipeName})

                                print(recipeName.encode())

                                fileName = "".join(
                                    [c for c in recipeName if re.match(r'\w', c)])
                                recipeData.update({'fileName': fileName})

                                # Load "Edit Page" to get propper values for
                                # Category / Makes / PrepTime / CookTime
                                recipeEditUrl = "{}recipeform.jsp?RECIPE_NAME={}".format(
                                    mealboard_url,
                                    recipeData['Name'])
                                content, session = getHtml(
                                    recipeEditUrl,
                                    session
                                )
                                if content is not None:
                                    recipeData = parseRecipeEditPage(
                                        content,
                                        recipeData
                                    )

                                # Download html page for recipe
                                recipeHtmlUrl = "{}recipeview.jsp?RECIPE_NAME={}".format(
                                    mealboard_url,
                                    recipeData['Name'])
                                recipeData.update({'sourceUrl': recipeHtmlUrl})

                                content, session = getHtml(
                                    recipeHtmlUrl, session)
                                if content is not None:

                                    # If we want to save the original, HTML recipe
                                    # TODO(Michael): Maybe make this a flag/option?
                                    if False:
                                        fileName = "./html/{0}.html".format(
                                            recipeData['fileName'])
                                        with open(fileName, 'w', encoding="utf-8") as fp:
                                            fp.write(content)

                                    # parse and convert recipe html page
                                    recipeHtml = parseRecipeHtml(
                                        content, recipeData)

                                    # Save new html page
                                    if not os.path.exists(location):
                                        os.makedirs(location)
                                    fileName = "{}/{}.html".format(
                                        location,
                                        recipeData['fileName']
                                    )
                                    with open(fileName, 'w', encoding='utf-8') as fp:
                                        fp.write(recipeHtml)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    user_default = 'MEALBOARD_USER env variable'
    pin_default = 'MEALBOARD_PIN env variable'

    parser.add_argument('-u',
                        action='store',
                        dest='user',
                        default=user_default,
                        help='Mealboard user to use to connect')
    parser.add_argument('-p',
                        action='store',
                        dest='pin',
                        default=pin_default,
                        help='Mealboard PIN to use to connect')
    parser.add_argument('-l',
                        action='store',
                        dest='location',
                        default='./recipes',
                        help='Location to store recipes')
    results = parser.parse_args()

    if (results.user == user_default):
        if ('MEALBOARD_USER' in os.environ):
            results.user = os.environ['MEALBOARD_USER']
        else:
            print('Mealboard user not set. Use the -u flag to set this, or '
                  'set the\nMEALBOARD_USER environment variable.')
            quit()
    if (results.pin == pin_default):
        if ('MEALBOARD_PIN' in os.environ):
            results.pin = os.environ['MEALBOARD_PIN']
        else:
            print('Mealboard pin not set. Use the -p flag to set this, or '
                  'set the\nMEALBOARD_PIN environment variable.')
            quit()

    main(results.user, results.pin, results.location)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import os
from datetime import datetime
from ChatExchange.chatexchange.client import Client
import HTMLParser
import md5
import ConfigParser

class GlobalVars:
    false_positives = []
    whitelisted_users = []
    blacklisted_users = []
    ignored_posts = []
    auto_ignored_posts = []
    startup_utc = datetime.utcnow().strftime("%H:%M:%S")
    latest_questions = []
    blockedTime = 0
    charcoal_room_id = "11540"
    meta_tavern_room_id = "89"
    socvr_room_id = "41570"
    site_filename = {"electronics.stackexchange.com": "ElectronicsGood.txt",
                     "gaming.stackexchange.com": "GamingGood.txt", "german.stackexchange.com": "GermanGood.txt",
                     "italian.stackexchange.com": "ItalianGood.txt", "math.stackexchange.com": "MathematicsGood.txt",
                     "spanish.stackexchange.com": "SpanishGood.txt", "stats.stackexchange.com": "StatsGood.txt"}

    experimental_reasons = ["Code block"]  # Don't widely report these

    parser = HTMLParser.HTMLParser()
    wrap = Client("stackexchange.com")
    wrapm = Client("meta.stackexchange.com")
    wrapso = Client("stackoverflow.com")
    privileged_users = {charcoal_room_id: ["117490",  # Normal Human
                                           "66258",  # Andy
                                           "31768",  # ManishEarth
                                           "103081",  # hichris123
                                           "73046",  # Undo
                                           "88521",  # ProgramFOX
                                           "59776",  # Doorknob
                                           "31465",  # Seth
                                           "88577",  # Santa Claus
                                           "34124",  # Andrew Leach
                                           "54229",  # apnorton
                                           "20459",  # S.L. Barth
                                           "32436"],  # tchrist
                        meta_tavern_room_id: ["259867",  # Normal Human
                                              "244519",  # Roombatron5000
                                              "244382",  # TGMCians
                                              "194047",  # Jan Dvorak
                                              "158100",  # rene
                                              "178438",  # Manishearth
                                              "237685",  # hichris123
                                              "215468",  # Undo
                                              "229438",  # ProgramFOX
                                              "180276",  # Doorknob
                                              "161974",  # Lynn Crumbling
                                              "186281",  # Andy
                                              "266094",  # Unihedro
                                              "245167",  # Infinite Recursion (No.)
                                              "230261",  # Jason C
                                              "213575",  # Braiam
                                              "241919",  # Andrew T.
                                              "203389",  # backwards-Seth
                                              "202832",  # Mooseman
                                              "160017",  # DragonLord the Fiery
                                              "201151",  # bummi
                                              "188558",  # Frank
                                              "229166",  # Santa Claus
                                              "159034",  # Kevin Brown
                                              "203972",  # PeterJ
                                              "188673",  # Alexis King
                                              "258672",  # AstroCB
                                              "227577",  # Sam
                                              "255735",  # cybermonkey
                                              "279182",  # Ixrec
                                              "271104",  # James
                                              "220428",  # Qantas 94 Heavy
                                              "153355",  # tchrist
                                              "238426",  # Ed Cottrell
                                              "166899",  # Second Rikudo
                                              "287999",  # ASCIIThenANSI
                                              "208518",  # JNat
                                              "284141",  # michaelpri
                                              "260312",  # vaultah
                                              "244062",  # SouravGhosh
                                              "152859",  # Shadow Wizard
                                              "201314",  # apnorton
                                              "280934",  # M.A.Ramezani
                                              "200235",  # durron597
                                              "148310",  # Awesome Poodles / Brock Adams
                                              "168333",  # S.L. Barth
                                              "257207",  # United States of America aka Unicorns are Very Very Yummy
                                              "244282",  # DroidDev
                                              "163250",  # Cupcake
                                              "298265",  # BoomsPlus
                                              "253560",  # josilber
                                              "244254",  # misterManSam
                                              "188189"],  # Robert Longson
                        socvr_room_id: [
                            "1849664",  # Undo
                            "2581872",  # hichris123
                            "1198729",  # Manishearth
                            "3717023",  # Normal Human aka 1999
                            "2619912",  # ProgramFOX
                            "578411",   # rene
                            "1043380",  # gunr2171
                            "2246344",  # Sam
                            "2756409",  # TylerH
                            "1768232",  # durron597
                            "359284",   # Kevin Brown
                            "258400",   # easwee
                            "3622940",  # Unihedron
                            "3204551",  # Deduplicator
                            "4342498",  # NathanOliver
                            "4639281",  # Tiny Giant
                            "3093387",  # josilber
                            "1652962",  # cimmanon
                            "1677912",  # Mogsdad
                            "656243",   # Lynn Crumbling
                            "3933332",  # Rizier123
                            "2422013",  # cybermonkey
                            "3478852",  # Nisse Engström
                            "2302862",  # Siguza
                            "1324",     # Paul Roub
                            "1743880",  # Tunaki
                            "1663001"]  # DavidG
                        }
    smokeDetector_user_id = {charcoal_room_id: "120914", meta_tavern_room_id: "266345",
                             socvr_room_id: "3735529"}

    censored_committer_names = {"3f4ed0f38df010ce300dba362fa63a62": "Undo1"}

    commit = os.popen('git log --pretty=format:"%h" -n 1').read()
    commit_author = os.popen('git log --pretty=format:"%cn" -n 1').read()

    if md5.new(commit_author).hexdigest() in censored_committer_names:
        commit_author = censored_committer_names[md5.new(commit_author).hexdigest()]

    commit_with_author = os.popen('git log --pretty=format:"%h (' + commit_author + ': *%s*)" -n 1').read()
    on_master = os.popen("git rev-parse --abbrev-ref HEAD").read().strip() == "master"
    charcoal_hq = None
    tavern_on_the_meta = None
    socvr = None
    s = ""
    s_reverted = ""
    specialrooms = []
    bayesian_testroom = None
    apiquota = -1
    bodyfetcher = None
    se_sites = []
    tavern_users_chatting = []
    frequent_sentences = []
    why_data = []
    notifications = []

    config = ConfigParser.RawConfigParser()
    config.read('config')

    latest_smokedetector_messages = {meta_tavern_room_id: [], charcoal_room_id: [],
                                     socvr_room_id: []}

    location = config.get("Config", "location")
    print location

    try:
        metasmoke_host = config.get("Config", "metasmoke_host")
        print metasmoke_host
    except ConfigParser.NoOptionError:
        metasmoke_host = None
        print "metasmoke host not found. Set it as metasmoke_host in the config file. See https://github.com/Charcoal-SE/metasmoke."

    try:
        metasmoke_key = config.get("Config", "metasmoke_key")
    except ConfigParser.NoOptionError:
        metasmoke_key = ""
        print "No metasmoke key found, which is okay if both are running on the same host"
<commit_msg>add JAL to priviledged users<commit_after># -*- coding: utf-8 -*-

import os
from datetime import datetime
from ChatExchange.chatexchange.client import Client
import HTMLParser
import md5
import ConfigParser

class GlobalVars:
    false_positives = []
    whitelisted_users = []
    blacklisted_users = []
    ignored_posts = []
    auto_ignored_posts = []
    startup_utc = datetime.utcnow().strftime("%H:%M:%S")
    latest_questions = []
    blockedTime = 0
    charcoal_room_id = "11540"
    meta_tavern_room_id = "89"
    socvr_room_id = "41570"
    site_filename = {"electronics.stackexchange.com": "ElectronicsGood.txt",
                     "gaming.stackexchange.com": "GamingGood.txt", "german.stackexchange.com": "GermanGood.txt",
                     "italian.stackexchange.com": "ItalianGood.txt", "math.stackexchange.com": "MathematicsGood.txt",
                     "spanish.stackexchange.com": "SpanishGood.txt", "stats.stackexchange.com": "StatsGood.txt"}

    experimental_reasons = ["Code block"]  # Don't widely report these

    parser = HTMLParser.HTMLParser()
    wrap = Client("stackexchange.com")
    wrapm = Client("meta.stackexchange.com")
    wrapso = Client("stackoverflow.com")
    privileged_users = {charcoal_room_id: ["117490",  # Normal Human
                                           "66258",  # Andy
                                           "31768",  # ManishEarth
                                           "103081",  # hichris123
                                           "73046",  # Undo
                                           "88521",  # ProgramFOX
                                           "59776",  # Doorknob
                                           "31465",  # Seth
                                           "88577",  # Santa Claus
                                           "34124",  # Andrew Leach
                                           "54229",  # apnorton
                                           "20459",  # S.L. Barth
                                           "32436"],  # tchrist
                        meta_tavern_room_id: ["259867",  # Normal Human
                                              "244519",  # Roombatron5000
                                              "244382",  # TGMCians
                                              "194047",  # Jan Dvorak
                                              "158100",  # rene
                                              "178438",  # Manishearth
                                              "237685",  # hichris123
                                              "215468",  # Undo
                                              "229438",  # ProgramFOX
                                              "180276",  # Doorknob
                                              "161974",  # Lynn Crumbling
                                              "186281",  # Andy
                                              "266094",  # Unihedro
                                              "245167",  # Infinite Recursion (No.)
                                              "230261",  # Jason C
                                              "213575",  # Braiam
                                              "241919",  # Andrew T.
                                              "203389",  # backwards-Seth
                                              "202832",  # Mooseman
                                              "160017",  # DragonLord the Fiery
                                              "201151",  # bummi
                                              "188558",  # Frank
                                              "229166",  # Santa Claus
                                              "159034",  # Kevin Brown
                                              "203972",  # PeterJ
                                              "188673",  # Alexis King
                                              "258672",  # AstroCB
                                              "227577",  # Sam
                                              "255735",  # cybermonkey
                                              "279182",  # Ixrec
                                              "271104",  # James
                                              "220428",  # Qantas 94 Heavy
                                              "153355",  # tchrist
                                              "238426",  # Ed Cottrell
                                              "166899",  # Second Rikudo
                                              "287999",  # ASCIIThenANSI
                                              "208518",  # JNat
                                              "284141",  # michaelpri
                                              "260312",  # vaultah
                                              "244062",  # SouravGhosh
                                              "152859",  # Shadow Wizard
                                              "201314",  # apnorton
                                              "280934",  # M.A.Ramezani
                                              "200235",  # durron597
                                              "148310",  # Awesome Poodles / Brock Adams
                                              "168333",  # S.L. Barth
                                              "257207",  # United States of America aka Unicorns are Very Very Yummy
                                              "244282",  # DroidDev
                                              "163250",  # Cupcake
                                              "298265",  # BoomsPlus
                                              "253560",  # josilber
                                              "244254",  # misterManSam
                                              "188189"],  # Robert Longson
                        socvr_room_id: [
                            "1849664",  # Undo
                            "2581872",  # hichris123
                            "1198729",  # Manishearth
                            "3717023",  # Normal Human aka 1999
                            "2619912",  # ProgramFOX
                            "578411",   # rene
                            "1043380",  # gunr2171
                            "2246344",  # Sam
                            "2756409",  # TylerH
                            "1768232",  # durron597
                            "359284",   # Kevin Brown
                            "258400",   # easwee
                            "3622940",  # Unihedron
                            "3204551",  # Deduplicator
                            "4342498",  # NathanOliver
                            "4639281",  # Tiny Giant
                            "3093387",  # josilber
                            "1652962",  # cimmanon
                            "1677912",  # Mogsdad
                            "656243",   # Lynn Crumbling
                            "3933332",  # Rizier123
                            "2422013",  # cybermonkey
                            "3478852",  # Nisse Engström
                            "2302862",  # Siguza
                            "1324",     # Paul Roub
                            "1743880",  # Tunaki
                            "1663001",  # DavidG
                            "2415822"]  # JAL
                        }
    smokeDetector_user_id = {charcoal_room_id: "120914", meta_tavern_room_id: "266345",
                             socvr_room_id: "3735529"}

    censored_committer_names = {"3f4ed0f38df010ce300dba362fa63a62": "Undo1"}

    commit = os.popen('git log --pretty=format:"%h" -n 1').read()
    commit_author = os.popen('git log --pretty=format:"%cn" -n 1').read()

    if md5.new(commit_author).hexdigest() in censored_committer_names:
        commit_author = censored_committer_names[md5.new(commit_author).hexdigest()]

    commit_with_author = os.popen('git log --pretty=format:"%h (' + commit_author + ': *%s*)" -n 1').read()
    on_master = os.popen("git rev-parse --abbrev-ref HEAD").read().strip() == "master"
    charcoal_hq = None
    tavern_on_the_meta = None
    socvr = None
    s = ""
    s_reverted = ""
    specialrooms = []
    bayesian_testroom = None
    apiquota = -1
    bodyfetcher = None
    se_sites = []
    tavern_users_chatting = []
    frequent_sentences = []
    why_data = []
    notifications = []

    config = ConfigParser.RawConfigParser()
    config.read('config')

    latest_smokedetector_messages = {meta_tavern_room_id: [], charcoal_room_id: [],
                                     socvr_room_id: []}

    location = config.get("Config", "location")
    print location

    try:
        metasmoke_host = config.get("Config", "metasmoke_host")
        print metasmoke_host
    except ConfigParser.NoOptionError:
        metasmoke_host = None
        print "metasmoke host not found. Set it as metasmoke_host in the config file. See https://github.com/Charcoal-SE/metasmoke."

    try:
        metasmoke_key = config.get("Config", "metasmoke_key")
    except ConfigParser.NoOptionError:
        metasmoke_key = ""
        print "No metasmoke key found, which is okay if both are running on the same host"
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from sqlalchemy_searchable import parse_search_query
from sqlalchemy_searchable.parser import SearchQueryParser

class TestParseSearchQuery(object):
    def test_uses_pgsql_wildcard_by_default(self):
        assert parse_search_query('star wars') == 'star:* & wars:*'

    def test_escapes_special_chars(self):
        assert parse_search_query('star!#') == 'star:*'

    def test_supports_custom_parsers(self):
        assert (
            parse_search_query(
                'star wars',
                parser=SearchQueryParser(wildcard='*')
            )
            ==
            'star* & wars*'
        )

    def test_operator_parsing(self):
        assert (
            parse_search_query('star or wars') ==
            'star:* | wars:*'
        )

    def test_empty_string(self):
        parse_search_query('') == ''

    def test_numbers(self):
        assert (
            parse_search_query('12331 or 12a12') ==
            '12331:* | 12a12:*'
        )

    def test_emails_without_email_tokens(self):
        assert (
            parse_search_query('john@fastmonkeys.com') ==
            'john:* & fastmonkeys:* & com:*'
        )

    def test_emails_with_email_tokens(self):
        assert (
            parse_search_query(
                'john@fastmonkeys.com',
                parser=SearchQueryParser(emails_as_tokens=True)
            ) ==
            'john@fastmonkeys.com:*'
        )

    def test_hyphen_between_words(self):
        assert parse_search_query('star-wars') == 'star:* & wars:*'

    def test_or(self):
        assert parse_search_query('star or wars') == 'star:* | wars:*'

    def test_multiple_ors(self):
        assert parse_search_query('star or or or wars') == 'star:* | wars:*'

    def test_space_as_and(self):
        assert parse_search_query('star wars') == 'star:* & wars:*'

    def test_multiple_spaces_as_and(self):
        assert (
            parse_search_query('star   wars    luke') ==
            'star:* & wars:* & luke:*'
        )

    def test_and(self):
        assert parse_search_query('star and wars') == 'star:* & wars:*'

    def test_multiple_and(self):
        assert parse_search_query('star and and wars') == 'star:* & wars:*'

    def test_parenthesis(self):
        assert parse_search_query('(star wars) or luke') == (
            '(star:* & wars:*) | luke:*'
        )

    def test_or_and(self):
        assert (
            parse_search_query('star or wars and luke or solo') ==
            'star:* | wars:* & luke:* | solo:*'
        )

    def test_empty_parenthesis(self):
        assert parse_search_query('()') == ''

    def test_nested_parenthesis(self):
        assert parse_search_query('((star wars)) or luke') == (
            '(star:* & wars:*) | luke:*'
        )

    def test_not(self):
        assert parse_search_query('-star') == (
            '! star:*'
        )

    def test_not_between_words(self):
        assert parse_search_query('wars -star') == (
            'wars:* & ! star:*'
        )
        assert parse_search_query(u'äää -ööö') == (
            u'äää:* & ! ööö:*'
        )

    def test_not_with_parenthesis(self):
        assert parse_search_query('-(star wars)') == '! (star:* & wars:*)'

    def test_double_quotes(self):
        assert parse_search_query('"star') == (
            'star:*'
        )

    def test_search_supports_non_english_characters(self):
        parse_search_query(u'ähtäri') == (
            u'ähtäri:*'
        )

    def test_single_quotes(self):
        assert parse_search_query("'star") == (
            'star:*'
        )
<commit_msg>Add failing test case, refs #22<commit_after># -*- coding: utf-8 -*-
from sqlalchemy_searchable import parse_search_query
from sqlalchemy_searchable.parser import SearchQueryParser

class TestParseSearchQuery(object):
    def test_uses_pgsql_wildcard_by_default(self):
        assert parse_search_query('star wars') == 'star:* & wars:*'

    def test_escapes_special_chars(self):
        assert parse_search_query('star!#') == 'star:*'

    def test_supports_custom_parsers(self):
        assert (
            parse_search_query(
                'star wars',
                parser=SearchQueryParser(wildcard='*')
            )
            ==
            'star* & wars*'
        )

    def test_operator_parsing(self):
        assert (
            parse_search_query('star or wars') ==
            'star:* | wars:*'
        )

    def test_empty_string(self):
        parse_search_query('') == ''

    def test_numbers(self):
        assert (
            parse_search_query('12331 or 12a12') ==
            '12331:* | 12a12:*'
        )

    def test_emails_without_email_tokens(self):
        assert (
            parse_search_query('john@fastmonkeys.com') ==
            'john:* & fastmonkeys:* & com:*'
        )

    def test_emails_with_email_tokens(self):
        assert (
            parse_search_query(
                'john@fastmonkeys.com',
                parser=SearchQueryParser(emails_as_tokens=True)
            ) ==
            'john@fastmonkeys.com:*'
        )

    def test_hyphen_between_words(self):
        assert parse_search_query('star-wars') == 'star:* & wars:*'

    def test_or(self):
        assert parse_search_query('star or wars') == 'star:* | wars:*'

    def test_multiple_ors(self):
        assert parse_search_query('star or or or wars') == 'star:* | wars:*'

    def test_space_as_and(self):
        assert parse_search_query('star wars') == 'star:* & wars:*'

    def test_multiple_spaces_as_and(self):
        assert (
            parse_search_query('star   wars    luke') ==
            'star:* & wars:* & luke:*'
        )

    def test_and(self):
        assert parse_search_query('star and wars') == 'star:* & wars:*'

    def test_multiple_and(self):
        assert parse_search_query('star and and wars') == 'star:* & wars:*'

    def test_parenthesis(self):
        assert parse_search_query('(star wars) or luke') == (
            '(star:* & wars:*) | luke:*'
        )

    def test_or_and(self):
        assert (
            parse_search_query('star or wars and luke or solo') ==
            'star:* | wars:* & luke:* | solo:*'
        )

    def test_empty_parenthesis(self):
        assert parse_search_query('()') == ''

    def test_nested_parenthesis(self):
        assert parse_search_query('((star wars)) or luke') == (
            '(star:* & wars:*) | luke:*'
        )

    def test_not(self):
        assert parse_search_query('-star') == (
            '! star:*'
        )

    def test_not_between_words(self):
        assert parse_search_query('wars -star') == (
            'wars:* & ! star:*'
        )
        assert parse_search_query(u'äää -ööö') == (
            u'äää:* & ! ööö:*'
        )

    def test_not_with_parenthesis(self):
        assert parse_search_query('-(star wars)') == '! (star:* & wars:*)'

    def test_double_quotes(self):
        assert parse_search_query('"star') == (
            'star:*'
        )

    def test_search_supports_non_english_characters(self):
        parse_search_query(u'ähtäri') == (
            u'ähtäri:*'
        )

    def test_single_quotes(self):
        assert parse_search_query("'star") == (
            'star:*'
        )

    def test_or_within_a_token(self):
        assert parse_search_query('organs') == (
            'organs:*'
        )

    def test_or_within_a_token_preceded_by_space(self):
        assert parse_search_query('star organs') == (
            'star:* & organs:*'
        )
<|endoftext|>


<commit_before># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

from videos.models import VIDEO_TYPE
from .videos.types import (
    video_type_registrar, UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION
)

# for now, they kind of match
ACCOUNT_TYPES = VIDEO_TYPE

class ThirdPartyAccountManager(models.Manager):

    def always_push_account(self):
        """
        Get the ThirdPartyAccount that is able to push to any video on Youtube.
        Raise ``ImproperlyConfigured`` if it can't be found.
        """
        username = getattr(settings, 'YOUTUBE_ALWAYS_PUSH_USERNAME')

        try:
            return self.get(username=username)
        except ThirdPartyAccount.DoesNotExist:
            raise ImproperlyConfigured("Can't find youtube account")

    def mirror_on_third_party(self, video, language, action, version=None):
        """
        Does the specified action (video.types.UPDATE_VERSION_ACTION or
                                   video.types.DELETE_LANGUAGE_ACTION)
        on the original account (e.g. Youtube video).
        For example, to update a given version to Youtube:
             ThirdPartyAccountManager.objects.mirror_on_third_party(
                       video, language, "update_subtitles", version)
        For deleting, we only delete languages, so it should be
              ThirdPartyAccountManager.objects.mirror_on_third_party(
                        video, language, "delete_subtitles")
        This method is 'safe' to call, meaning that we only do syncing if there
        are matching third party credentials for this video.
        The update will only be done if the version is synced
        """
        if action not in [UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION]:
            raise NotImplementedError(
                "Mirror to third party does not support the %s action" % action)

        if version:
            if not version.is_public or not version.is_synced():
                # We can't mirror unsynced or non-public versions.
                return

        rule = YoutubeSyncRule.objects.all()[0]
        should_sync = rule.should_sync(video)
        always_push_account = self.always_push_account()

        for vurl in video.videourl_set.all():
            already_updated = False
            vt = video_type_registrar.video_type_for_url(vurl.url)

            if should_sync:
                vt.update_subtitles(version, always_push_account)
                already_updated = False

            username = vurl.owner_username

            if not username:
                continue
            try:
                account = ThirdPartyAccount.objects.get(type=vurl.type, username=username)
            except ThirdPartyAccount.DoesNotExist:
                continue

            if hasattr(vt, action):
                if action == UPDATE_VERSION_ACTION and not already_updated:
                    vt.update_subtitles(version, account)
                elif action == DELETE_LANGUAGE_ACTION:
                    vt.delete_subtitles(language, account)

class ThirdPartyAccount(models.Model):
    """
    Links a third party account (e.g. YouTube's') to a certain video URL
    This allows us to push changes in Unisubs back to that video provider.
    The user links a video on unisubs to his youtube account. Once edits to
    any languages are done, we push those back to Youtube.
    For know, this only supports Youtube, but nothing is stopping it from
    working with others.
    """
    type = models.CharField(max_length=10, choices=ACCOUNT_TYPES)
    # this is the third party account user name, eg the youtube user
    username  = models.CharField(max_length=255, db_index=True,
                                 null=False, blank=False)
    oauth_access_token = models.CharField(max_length=255, db_index=True,
                                          null=False, blank=False)
    oauth_refresh_token = models.CharField(max_length=255, db_index=True,
                                           null=False, blank=False)

    objects = ThirdPartyAccountManager()

    class Meta:
        unique_together = ("type", "username")

    def __unicode__(self):
        return '%s - %s' % (self.get_type_display(), self.username)

class YoutubeSyncRule(models.Model):
    team = models.TextField(default='', blank=True,
            help_text='Comma separated list of slugs')
    user = models.TextField(default='', blank=True,
            help_text='Comma separated list of usernames')
    video = models.TextField(default='', blank=True,
            help_text='Comma separated list of pks')

    def __unicode__(self):
        return 'Youtube sync rule'

    def team_in_list(self, team):
        teams = self.team.split(',')
        if '*' in teams:
            return True
        return team in teams

    def user_in_list(self, user):
        users = self.user.split(',')
        if '*' in users:
            return True
        return user.username in users

    def video_in_list(self, pk):
        pks = self.video.split(',')
        if '*' in pks:
            return True
        if len(pks) == 1 and pks[0] == '':
            return False
        return pk in map(int, pks)

    def should_sync(self, video):
        return self.team_in_list(video.get_team_video().team.slug) or \
                self.user_in_list(video.user) or \
                self.video_in_list(video.pk)
<commit_msg>Document youtube rules<commit_after># Amara, universalsubtitles.org
#
# Copyright (C) 2012 Participatory Culture Foundation
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see
# http://www.gnu.org/licenses/agpl-3.0.html.

from django.db import models
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

from videos.models import VIDEO_TYPE
from .videos.types import (
    video_type_registrar, UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION
)

# for now, they kind of match
ACCOUNT_TYPES = VIDEO_TYPE

class ThirdPartyAccountManager(models.Manager):

    def always_push_account(self):
        """
        Get the ThirdPartyAccount that is able to push to any video on Youtube.
        Raise ``ImproperlyConfigured`` if it can't be found.
        """
        username = getattr(settings, 'YOUTUBE_ALWAYS_PUSH_USERNAME')

        try:
            return self.get(username=username)
        except ThirdPartyAccount.DoesNotExist:
            raise ImproperlyConfigured("Can't find youtube account")

    def mirror_on_third_party(self, video, language, action, version=None):
        """
        Does the specified action (video.types.UPDATE_VERSION_ACTION or
                                   video.types.DELETE_LANGUAGE_ACTION)
        on the original account (e.g. Youtube video).
        For example, to update a given version to Youtube:
             ThirdPartyAccountManager.objects.mirror_on_third_party(
                       video, language, "update_subtitles", version)
        For deleting, we only delete languages, so it should be
              ThirdPartyAccountManager.objects.mirror_on_third_party(
                        video, language, "delete_subtitles")
        This method is 'safe' to call, meaning that we only do syncing if there
        are matching third party credentials for this video.
        The update will only be done if the version is synced
        """
        if action not in [UPDATE_VERSION_ACTION, DELETE_LANGUAGE_ACTION]:
            raise NotImplementedError(
                "Mirror to third party does not support the %s action" % action)

        if version:
            if not version.is_public or not version.is_synced():
                # We can't mirror unsynced or non-public versions.
                return

        rule = YoutubeSyncRule.objects.all()[0]
        should_sync = rule.should_sync(video)
        always_push_account = self.always_push_account()

        for vurl in video.videourl_set.all():
            already_updated = False
            vt = video_type_registrar.video_type_for_url(vurl.url)

            if should_sync:
                vt.update_subtitles(version, always_push_account)
                already_updated = False

            username = vurl.owner_username

            if not username:
                continue
            try:
                account = ThirdPartyAccount.objects.get(type=vurl.type, username=username)
            except ThirdPartyAccount.DoesNotExist:
                continue

            if hasattr(vt, action):
                if action == UPDATE_VERSION_ACTION and not already_updated:
                    vt.update_subtitles(version, account)
                elif action == DELETE_LANGUAGE_ACTION:
                    vt.delete_subtitles(language, account)

class ThirdPartyAccount(models.Model):
    """
    Links a third party account (e.g. YouTube's') to a certain video URL
    This allows us to push changes in Unisubs back to that video provider.
    The user links a video on unisubs to his youtube account. Once edits to
    any languages are done, we push those back to Youtube.
    For know, this only supports Youtube, but nothing is stopping it from
    working with others.
    """
    type = models.CharField(max_length=10, choices=ACCOUNT_TYPES)
    # this is the third party account user name, eg the youtube user
    username  = models.CharField(max_length=255, db_index=True,
                                 null=False, blank=False)
    oauth_access_token = models.CharField(max_length=255, db_index=True,
                                          null=False, blank=False)
    oauth_refresh_token = models.CharField(max_length=255, db_index=True,
                                           null=False, blank=False)

    objects = ThirdPartyAccountManager()

    class Meta:
        unique_together = ("type", "username")

    def __unicode__(self):
        return '%s - %s' % (self.get_type_display(), self.username)

class YoutubeSyncRule(models.Model):
    """
    An instance of this class determines which Youtube videos should be synced
    back to Youtube via the new integration.

    There should only ever be one instance of this class in the database.

    You should run a query and then call it like this:

        rule = YoutubeSyncRule.objects.all()[0]
        rule.should_sync(video)

    Where ``video`` is a ``videos.models.Video`` instance.

    ``team`` should be a comma-separated list of team slugs that you want to
    sync.  ``user`` should be a comma-separated list of usernames of users
    whose videos should be synced.  ``video`` is a list of primary keys of
    videos that should be synced.

    You can also specify a wildcard "*" to any of the above to match any teams,
    any users, or any videos.
    """
    team = models.TextField(default='', blank=True,
            help_text='Comma separated list of slugs')
    user = models.TextField(default='', blank=True,
            help_text='Comma separated list of usernames')
    video = models.TextField(default='', blank=True,
            help_text='Comma separated list of pks')

    def __unicode__(self):
        return 'Youtube sync rule'

    def team_in_list(self, team):
        teams = self.team.split(',')
        if '*' in teams:
            return True
        return team in teams

    def user_in_list(self, user):
        users = self.user.split(',')
        if '*' in users:
            return True
        return user.username in users

    def video_in_list(self, pk):
        pks = self.video.split(',')
        if '*' in pks:
            return True
        if len(pks) == 1 and pks[0] == '':
            return False
        return pk in map(int, pks)

    def should_sync(self, video):
        return self.team_in_list(video.get_team_video().team.slug) or \
                self.user_in_list(video.user) or \
                self.video_in_list(video.pk)
<|endoftext|>