<commit_before># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import contextlib
import os
import sys

from py_utils import tempfile_ext

from telemetry import benchmark
from telemetry import story
from telemetry.core import util
from telemetry import decorators
from telemetry.page import page as page_module
from telemetry.page import legacy_page_test
from telemetry.project_config import ProjectConfig
from telemetry import record_wpr
from telemetry.testing import options_for_unittests
from telemetry.testing import tab_test_case
from telemetry.util import wpr_modes

class MockPage(page_module.Page):
  def __init__(self, story_set, url):
    super(MockPage, self).__init__(url=url,
                                   page_set=story_set,
                                   base_dir=util.GetUnittestDataDir(),
                                   name=url)
    self.func_calls = []

  def RunPageInteractions(self, _):
    self.func_calls.append('RunPageInteractions')

  def RunSmoothness(self, _):
    self.func_calls.append('RunSmoothness')

class MockStorySet(story.StorySet):
  def __init__(self, url=''):
    super(MockStorySet, self).__init__(
        archive_data_file='data/archive_files/test.json')
    self.AddStory(MockPage(self, url))

class MockPageTest(legacy_page_test.LegacyPageTest):
  def __init__(self):
    super(MockPageTest, self).__init__()
    self._action_name_to_run = "RunPageInteractions"
    self.func_calls = []

  def CustomizeBrowserOptions(self, options):
    self.func_calls.append('CustomizeBrowserOptions')

  def WillNavigateToPage(self, page, tab):
    self.func_calls.append('WillNavigateToPage')

  def DidNavigateToPage(self, page, tab):
    self.func_calls.append('DidNavigateToPage')

  def ValidateAndMeasurePage(self, page, tab, results):
    self.func_calls.append('ValidateAndMeasurePage')

  def WillStartBrowser(self, platform):
    self.func_calls.append('WillStartBrowser')

  def DidStartBrowser(self, browser):
    self.func_calls.append('DidStartBrowser')

class MockBenchmark(benchmark.Benchmark):
  test = MockPageTest

  def __init__(self):
    super(MockBenchmark, self).__init__()
    self.mock_story_set = None

  @classmethod
  def AddBenchmarkCommandLineArgs(cls, group):
    group.add_option('', '--mock-benchmark-url', action='store', type='string')

  def CreateStorySet(self, options):
    kwargs = {}
    if options.mock_benchmark_url:
      kwargs['url'] = options.mock_benchmark_url
    self.mock_story_set = MockStorySet(**kwargs)
    return self.mock_story_set

  def SetExtraBrowserOptions(self, browser_options):
    browser_options.AppendExtraBrowserArgs('--extra-option')

class MockTimelineBasedMeasurementBenchmark(benchmark.Benchmark):

  def __init__(self):
    super(MockTimelineBasedMeasurementBenchmark, self).__init__()
    self.mock_story_set = None

  @classmethod
  def AddBenchmarkCommandLineArgs(cls, group):
    group.add_option('', '--mock-benchmark-url', action='store', type='string')

  def CreateStorySet(self, options):
    kwargs = {}
    if options.mock_benchmark_url:
      kwargs['url'] = options.mock_benchmark_url
    self.mock_story_set = MockStorySet(**kwargs)
    return self.mock_story_set

def _SuccessfulStories(results):
  return sorted(set(run.story for run in results.IterStoryRuns() if run.ok))

def _SkippedStories(results):
  return sorted(set(run.story
                    for run in results.IterStoryRuns() if run.skipped))

class RecordWprUnitTests(tab_test_case.TabTestCase):

  _base_dir = util.GetUnittestDataDir()
  _test_data_dir = os.path.join(util.GetUnittestDataDir(), 'page_tests')

  @classmethod
  def setUpClass(cls):
    sys.path.extend([cls._base_dir, cls._test_data_dir])
    super(RecordWprUnitTests, cls).setUpClass()
    cls._url = cls.UrlOfUnittestFile('blank.html')
    cls._test_options = options_for_unittests.GetCopy()

  @contextlib.contextmanager
  def SkippingWith(self, expectations):
    with tempfile_ext.NamedTemporaryFile(prefix='expectations_',
                                         suffix='.config',
                                         mode='w+') as expectations_out:
      expectations_out.write('# results: [ Skip ]\n')
      for issue, abenchmark, astory in expectations:
        expectations_out.write('{} {}/{} [ Skip ]\n'.format(
            issue, abenchmark, astory))
      expectations_out.close()
      yield expectations_out.name

  # When the RecorderPageTest is created from a Benchmark, the benchmark will
  # have a PageTest, specified by its test attribute.
  def testRunPage_OnlyRunBenchmarkAction(self):
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)

  def testRunPage_CallBenchmarksPageTestsFunctions(self):
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)
    self.assertEqual(1, len(page_test.func_calls))
    self.assertEqual('ValidateAndMeasurePage', page_test.func_calls[0])

  def GetBrowserDeviceFlags(self):
    flags = ['--browser', self._browser.browser_type,
             '--remote', self._test_options.cros_remote,
             '--device', self._device]
    if self._browser.browser_type == 'exact':
      flags += ['--browser-executable', self._test_options.browser_executable]
    if self._test_options.chrome_root:
      flags += ['--chrome-root', self._test_options.chrome_root]
    return flags

  @decorators.Disabled('chromeos') # crbug.com/404868.
  def testWprRecorderWithPageSet(self):
    flags = self.GetBrowserDeviceFlags()
    mock_story_set = MockStorySet(url=self._url)
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_story_set, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_story_set.stories), _SuccessfulStories(results))

  def testWprRecorderWithBenchmark(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testWprRecorderWithTimelineBasedMeasurementBenchmark(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockTimelineBasedMeasurementBenchmark()
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testPageSetBaseDirFlag(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--page-set-base-dir', self._test_data_dir,
                  '--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with record_wpr.WprRecorder(ProjectConfig('non-existent-dummy-dir'),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testCommandLineFlags(self):
    flags = [
        '--pageset-repeat', '2',
        '--mock-benchmark-url', self._url,
        '--upload',
    ]
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      # page_runner command-line args
      self.assertEqual(2, wpr_recorder.options.pageset_repeat)
      # benchmark command-line args
      self.assertEqual(self._url, wpr_recorder.options.mock_benchmark_url)
      # record_wpr command-line arg to upload to cloud-storage.
      self.assertTrue(wpr_recorder.options.upload)
      # --extra-option added from Benchmark.SetExtraBrowserOptions()
      self.assertTrue('--extra-option' in
                      wpr_recorder.options.browser_options.extra_browser_args)
      # invalid command-line args
      self.assertFalse(hasattr(wpr_recorder.options, 'not_a_real_option'))

  def testRecordingEnabled(self):
    flags = ['--mock-benchmark-url', self._url]
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      self.assertEqual(wpr_modes.WPR_RECORD,
                       wpr_recorder.options.browser_options.wpr_mode)

  # When the RecorderPageTest CustomizeBrowserOptions/WillStartBrowser/
  # DidStartBrowser function is called, it forwards the call to the PageTest
  def testRecorderPageTest_BrowserMethods(self):
    flags = ['--mock-benchmark-url', self._url]
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      record_page_test.CustomizeBrowserOptions(wpr_recorder.options)
      record_page_test.WillStartBrowser(self._tab.browser.platform)
      record_page_test.DidStartBrowser(self._tab.browser)
      self.assertTrue('CustomizeBrowserOptions' in page_test.func_calls)
      self.assertTrue('WillStartBrowser' in page_test.func_calls)
      self.assertTrue('DidStartBrowser' in page_test.func_calls)

  def testUseLiveSitesUnsupported(self):
    flags = ['--use-live-sites']
    with self.assertRaises(SystemExit):
      record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                             MockBenchmark(), flags)

  def testWprRecorderWithBenchmarkAndExpectations(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    skips = [('crbug.com/1237031', mock_benchmark.Name(), page.name)]
    with self.SkippingWith(expectations=skips) as expectations_file:
      config = ProjectConfig(self._test_data_dir,
                             expectations_files=[expectations_file])
      with record_wpr.WprRecorder(config, mock_benchmark,
                                  flags) as wpr_recorder:
        with wpr_recorder.CreateResults() as results:
          wpr_recorder.Record(results)
          self.assertEqual([], _SuccessfulStories(results))
          self.assertEqual(
              sorted(mock_benchmark.mock_story_set.stories),
              _SkippedStories(results))

  def testWprRecorderWithBenchmarkAndEmptyExpectations(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with self.SkippingWith(expectations=[]) as expectations_file:
      config = ProjectConfig(self._test_data_dir,
                             expectations_files=[expectations_file])
      with record_wpr.WprRecorder(config, mock_benchmark,
                                  flags) as wpr_recorder:
        with wpr_recorder.CreateResults() as results:
          wpr_recorder.Record(results)
          self.assertEqual([], _SkippedStories(results))
          self.assertEqual(
              sorted(mock_benchmark.mock_story_set.stories),
              _SuccessfulStories(results))
<commit_msg>Add unit test for skipping flags in the RecordWpr ctor<commit_after># Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from __future__ import absolute_import
import contextlib
import os
import sys

from py_utils import tempfile_ext

from telemetry import benchmark
from telemetry import story
from telemetry.core import util
from telemetry import decorators
from telemetry.page import page as page_module
from telemetry.page import legacy_page_test
from telemetry.project_config import ProjectConfig
from telemetry import record_wpr
from telemetry.testing import options_for_unittests
from telemetry.testing import tab_test_case
from telemetry.util import wpr_modes

class MockPage(page_module.Page):
  def __init__(self, story_set, url):
    super(MockPage, self).__init__(url=url,
                                   page_set=story_set,
                                   base_dir=util.GetUnittestDataDir(),
                                   name=url)
    self.func_calls = []

  def RunPageInteractions(self, _):
    self.func_calls.append('RunPageInteractions')

  def RunSmoothness(self, _):
    self.func_calls.append('RunSmoothness')

class MockStorySet(story.StorySet):
  def __init__(self, url=''):
    if url == '':
      return
    super(MockStorySet, self).__init__(
        archive_data_file='data/archive_files/test.json')
    self.AddStory(MockPage(self, url))

class MockPageTest(legacy_page_test.LegacyPageTest):
  def __init__(self):
    super(MockPageTest, self).__init__()
    self._action_name_to_run = "RunPageInteractions"
    self.func_calls = []

  def CustomizeBrowserOptions(self, options):
    self.func_calls.append('CustomizeBrowserOptions')

  def WillNavigateToPage(self, page, tab):
    self.func_calls.append('WillNavigateToPage')

  def DidNavigateToPage(self, page, tab):
    self.func_calls.append('DidNavigateToPage')

  def ValidateAndMeasurePage(self, page, tab, results):
    self.func_calls.append('ValidateAndMeasurePage')

  def WillStartBrowser(self, platform):
    self.func_calls.append('WillStartBrowser')

  def DidStartBrowser(self, browser):
    self.func_calls.append('DidStartBrowser')

class MockBenchmark(benchmark.Benchmark):
  test = MockPageTest

  def __init__(self):
    super(MockBenchmark, self).__init__()
    self.mock_story_set = None

  @classmethod
  def AddBenchmarkCommandLineArgs(cls, group):
    group.add_option('', '--mock-benchmark-url', action='store', type='string')

  def CreateStorySet(self, options):
    kwargs = {}
    if hasattr(options, 'mock_benchmark_url') and options.mock_benchmark_url:
      kwargs['url'] = options.mock_benchmark_url
    self.mock_story_set = MockStorySet(**kwargs)
    return self.mock_story_set

  def SetExtraBrowserOptions(self, browser_options):
    browser_options.AppendExtraBrowserArgs('--extra-option')

class MockTimelineBasedMeasurementBenchmark(benchmark.Benchmark):

  def __init__(self):
    super(MockTimelineBasedMeasurementBenchmark, self).__init__()
    self.mock_story_set = None

  @classmethod
  def AddBenchmarkCommandLineArgs(cls, group):
    group.add_option('', '--mock-benchmark-url', action='store', type='string')

  def CreateStorySet(self, options):
    kwargs = {}
    if options.mock_benchmark_url:
      kwargs['url'] = options.mock_benchmark_url
    self.mock_story_set = MockStorySet(**kwargs)
    return self.mock_story_set

def _SuccessfulStories(results):
  return sorted(set(run.story for run in results.IterStoryRuns() if run.ok))

def _SkippedStories(results):
  return sorted(set(run.story
                    for run in results.IterStoryRuns() if run.skipped))

class RecordWprUnitTests(tab_test_case.TabTestCase):

  _base_dir = util.GetUnittestDataDir()
  _test_data_dir = os.path.join(util.GetUnittestDataDir(), 'page_tests')

  @classmethod
  def setUpClass(cls):
    sys.path.extend([cls._base_dir, cls._test_data_dir])
    super(RecordWprUnitTests, cls).setUpClass()
    cls._url = cls.UrlOfUnittestFile('blank.html')
    cls._test_options = options_for_unittests.GetCopy()

  @contextlib.contextmanager
  def SkippingWith(self, expectations):
    with tempfile_ext.NamedTemporaryFile(prefix='expectations_',
                                         suffix='.config',
                                         mode='w+') as expectations_out:
      expectations_out.write('# results: [ Skip ]\n')
      for issue, abenchmark, astory in expectations:
        expectations_out.write('{} {}/{} [ Skip ]\n'.format(
            issue, abenchmark, astory))
      expectations_out.close()
      yield expectations_out.name

  # When the RecorderPageTest is created from a Benchmark, the benchmark will
  # have a PageTest, specified by its test attribute.
  def testRunPage_OnlyRunBenchmarkAction(self):
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)

  def testRunPage_CallBenchmarksPageTestsFunctions(self):
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    record_page_test.ValidateAndMeasurePage(page, self._tab, results=None)
    self.assertEqual(1, len(page_test.func_calls))
    self.assertEqual('ValidateAndMeasurePage', page_test.func_calls[0])

  def GetBrowserDeviceFlags(self):
    flags = ['--browser', self._browser.browser_type,
             '--remote', self._test_options.cros_remote,
             '--device', self._device]
    if self._browser.browser_type == 'exact':
      flags += ['--browser-executable', self._test_options.browser_executable]
    if self._test_options.chrome_root:
      flags += ['--chrome-root', self._test_options.chrome_root]
    return flags

  @decorators.Disabled('chromeos') # crbug.com/404868.
  def testWprRecorderWithPageSet(self):
    flags = self.GetBrowserDeviceFlags()
    mock_story_set = MockStorySet(url=self._url)
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_story_set, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_story_set.stories), _SuccessfulStories(results))

  def testWprRecorderWithBenchmark(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testWprRecorderWithTimelineBasedMeasurementBenchmark(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockTimelineBasedMeasurementBenchmark()
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testPageSetBaseDirFlag(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--page-set-base-dir', self._test_data_dir,
                  '--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with record_wpr.WprRecorder(ProjectConfig('non-existent-dummy-dir'),
                                mock_benchmark, flags) as wpr_recorder:
      with wpr_recorder.CreateResults() as results:
        wpr_recorder.Record(results)
        self.assertEqual(
            sorted(mock_benchmark.mock_story_set.stories),
            _SuccessfulStories(results))

  def testCommandLineFlags(self):
    flags = [
        '--pageset-repeat', '2',
        '--mock-benchmark-url', self._url,
        '--upload',
    ]
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      # page_runner command-line args
      self.assertEqual(2, wpr_recorder.options.pageset_repeat)
      # benchmark command-line args
      self.assertEqual(self._url, wpr_recorder.options.mock_benchmark_url)
      # record_wpr command-line arg to upload to cloud-storage.
      self.assertTrue(wpr_recorder.options.upload)
      # --extra-option added from Benchmark.SetExtraBrowserOptions()
      self.assertTrue('--extra-option' in
                      wpr_recorder.options.browser_options.extra_browser_args)
      # invalid command-line args
      self.assertFalse(hasattr(wpr_recorder.options, 'not_a_real_option'))

  def testCommandLineFlagParsingSkipped(self):
    flags = [
        '--pageset-repeat', '2',
        '--mock-benchmark-url', self._url,
        '--upload',
    ]
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags, False) as wpr_recorder:
      self.assertFalse(hasattr(wpr_recorder.options, 'mock_benchmark_url'))

  def testRecordingEnabled(self):
    flags = ['--mock-benchmark-url', self._url]
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      self.assertEqual(wpr_modes.WPR_RECORD,
                       wpr_recorder.options.browser_options.wpr_mode)

  # When the RecorderPageTest CustomizeBrowserOptions/WillStartBrowser/
  # DidStartBrowser function is called, it forwards the call to the PageTest
  def testRecorderPageTest_BrowserMethods(self):
    flags = ['--mock-benchmark-url', self._url]
    page_test = MockBenchmark().test()
    record_page_test = record_wpr.RecorderPageTest(page_test)
    with record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                                MockBenchmark(), flags) as wpr_recorder:
      record_page_test.CustomizeBrowserOptions(wpr_recorder.options)
      record_page_test.WillStartBrowser(self._tab.browser.platform)
      record_page_test.DidStartBrowser(self._tab.browser)
      self.assertTrue('CustomizeBrowserOptions' in page_test.func_calls)
      self.assertTrue('WillStartBrowser' in page_test.func_calls)
      self.assertTrue('DidStartBrowser' in page_test.func_calls)

  def testUseLiveSitesUnsupported(self):
    flags = ['--use-live-sites']
    with self.assertRaises(SystemExit):
      record_wpr.WprRecorder(ProjectConfig(self._test_data_dir),
                             MockBenchmark(), flags)

  def testWprRecorderWithBenchmarkAndExpectations(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    page = MockPage(story_set=MockStorySet(url=self._url), url=self._url)
    skips = [('crbug.com/1237031', mock_benchmark.Name(), page.name)]
    with self.SkippingWith(expectations=skips) as expectations_file:
      config = ProjectConfig(self._test_data_dir,
                             expectations_files=[expectations_file])
      with record_wpr.WprRecorder(config, mock_benchmark,
                                  flags) as wpr_recorder:
        with wpr_recorder.CreateResults() as results:
          wpr_recorder.Record(results)
          self.assertEqual([], _SuccessfulStories(results))
          self.assertEqual(
              sorted(mock_benchmark.mock_story_set.stories),
              _SkippedStories(results))

  def testWprRecorderWithBenchmarkAndEmptyExpectations(self):
    flags = self.GetBrowserDeviceFlags()
    flags.extend(['--mock-benchmark-url', self._url])
    mock_benchmark = MockBenchmark()
    with self.SkippingWith(expectations=[]) as expectations_file:
      config = ProjectConfig(self._test_data_dir,
                             expectations_files=[expectations_file])
      with record_wpr.WprRecorder(config, mock_benchmark,
                                  flags) as wpr_recorder:
        with wpr_recorder.CreateResults() as results:
          wpr_recorder.Record(results)
          self.assertEqual([], _SkippedStories(results))
          self.assertEqual(
              sorted(mock_benchmark.mock_story_set.stories),
              _SuccessfulStories(results))
<|endoftext|>


<commit_before>##
##  patts-qt - Qt GUI client for PATTS
##  Copyright (C) 2015 Delwink, LLC
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as published by
##  the Free Software Foundation, version 3 only.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

from PyQt4.QtGui import QDialog, QPushButton, QHBoxLayout, QVBoxLayout
from PyQt4.QtGui import QLineEdit, QDesktopWidget, QLabel, QPalette, QCheckBox
from PyQt4.QtCore import Qt

from .lang import _
from .config import get, put

class LoginWindow(QDialog):
    def __init__(self, message='', logInText='LoginWindow.logIn',
                 cancelText='cancel'):
        super().__init__()

        self._cancelled = False
        self._ready = False

        logInButton = QPushButton(_(logInText))
        logInButton.clicked.connect(self._store_results)
        logInButton.setDefault(True)

        cancelButton = QPushButton(_(cancelText))
        cancelButton.clicked.connect(self._cancel)

        buttonBox = QHBoxLayout()
        buttonBox.addStretch(1)
        buttonBox.addWidget(cancelButton)
        buttonBox.addWidget(logInButton)

        self.serverInput = QLineEdit(self)
        self.serverInput.setPlaceholderText(_('LoginWindow.server'))
        self.serverInput.setToolTip(_('LoginWindow.serverToolTip'))

        self.dbInput = QLineEdit(self)
        self.dbInput.setPlaceholderText(_('LoginWindow.database'))
        self.dbInput.setToolTip(_('LoginWindow.databaseToolTip'))

        self.userInput = QLineEdit(self)
        self.userInput.setPlaceholderText(_('LoginWindow.user'))
        self.userInput.setToolTip(_('LoginWindow.userToolTip'))

        self.passInput = QLineEdit(self)
        self.passInput.setPlaceholderText(_('LoginWindow.password'))
        self.passInput.setToolTip(_('LoginWindow.passwordToolTip'))
        self.passInput.setEchoMode(QLineEdit.Password)

        self.autoLogin = QCheckBox()
        self.autoLogin.setText(_('LoginWindow.autoLogin'))
        self.autoLogin.toggled.connect(self._set_auto)

        self.serverInput.setText(get('Login', 'host'))
        self.dbInput.setText(get('Login', 'database'))
        self.userInput.setText(get('Login', 'user'))

        if not self.serverInput.text():
            self.serverInput.setFocus()
        elif not self.dbInput.text():
            self.dbInput.setFocus()
        elif not self.userInput.text():
            self.userInput.setFocus()
        else:
            self.passInput.setFocus()

        if message:
            self.errorLabel = QLabel(_(message))
            palette = QPalette()
            palette.setColor(QPalette.Foreground, Qt.red)
            self.errorLabel.setPalette(palette)

        inputBox = QVBoxLayout()
        if message:
            inputBox.addWidget(self.errorLabel)
        inputBox.addWidget(self.serverInput)
        inputBox.addWidget(self.dbInput)
        inputBox.addWidget(self.userInput)
        inputBox.addWidget(self.passInput)
        inputBox.addWidget(self.autoLogin)

        winLayout = QVBoxLayout()
        winLayout.addLayout(inputBox)
        winLayout.addLayout(buttonBox)

        self.setLayout(winLayout)
        self.setWindowTitle(_('LoginWindow.title'))
        self.resize(250, 200)
        self._center()

    def _store_results(self):
        self.hide()
        self.deleteLater()
        self._server = self.serverInput.text()
        self._db = self.dbInput.text()
        self._user = self.userInput.text()
        self._pass = self.passInput.text()
        self._ready = True

    def _cancel(self):
        self.hide()
        self.deleteLater()
        self._cancelled = True
        self._ready = True

    def _set_auto(self):
        if self.autoLogin.isChecked():
            put('Login', 'autologin', 'true')
        else:
            put('Login', 'autologin', 'false')

    def _center(self):
        geom = self.frameGeometry()
        centerPoint = QDesktopWidget().availableGeometry().center()
        geom.moveCenter(centerPoint)
        self.move(geom.topLeft())

    def get_info(self):
        self.exec_()
        if self._cancelled or not self._ready:
            return (None, None, None, None)

        return (self._user, self._pass, self._server, self._db)

def get_login(message=''):
    l = LoginWindow(message)
    return l.get_info()
<commit_msg>Set ready to false each time the login dialog is called<commit_after>##
##  patts-qt - Qt GUI client for PATTS
##  Copyright (C) 2015-2016 Delwink, LLC
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as published by
##  the Free Software Foundation, version 3 only.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.
##

from PyQt4.QtGui import QDialog, QPushButton, QHBoxLayout, QVBoxLayout
from PyQt4.QtGui import QLineEdit, QDesktopWidget, QLabel, QPalette, QCheckBox
from PyQt4.QtCore import Qt

from .lang import _
from .config import get, put

class LoginWindow(QDialog):
    def __init__(self, message='', logInText='LoginWindow.logIn',
                 cancelText='cancel'):
        super().__init__()

        self._cancelled = False
        self._ready = False

        logInButton = QPushButton(_(logInText))
        logInButton.clicked.connect(self._store_results)
        logInButton.setDefault(True)

        cancelButton = QPushButton(_(cancelText))
        cancelButton.clicked.connect(self._cancel)

        buttonBox = QHBoxLayout()
        buttonBox.addStretch(1)
        buttonBox.addWidget(cancelButton)
        buttonBox.addWidget(logInButton)

        self.serverInput = QLineEdit(self)
        self.serverInput.setPlaceholderText(_('LoginWindow.server'))
        self.serverInput.setToolTip(_('LoginWindow.serverToolTip'))

        self.dbInput = QLineEdit(self)
        self.dbInput.setPlaceholderText(_('LoginWindow.database'))
        self.dbInput.setToolTip(_('LoginWindow.databaseToolTip'))

        self.userInput = QLineEdit(self)
        self.userInput.setPlaceholderText(_('LoginWindow.user'))
        self.userInput.setToolTip(_('LoginWindow.userToolTip'))

        self.passInput = QLineEdit(self)
        self.passInput.setPlaceholderText(_('LoginWindow.password'))
        self.passInput.setToolTip(_('LoginWindow.passwordToolTip'))
        self.passInput.setEchoMode(QLineEdit.Password)

        self.autoLogin = QCheckBox()
        self.autoLogin.setText(_('LoginWindow.autoLogin'))
        self.autoLogin.toggled.connect(self._set_auto)

        self.serverInput.setText(get('Login', 'host'))
        self.dbInput.setText(get('Login', 'database'))
        self.userInput.setText(get('Login', 'user'))

        if not self.serverInput.text():
            self.serverInput.setFocus()
        elif not self.dbInput.text():
            self.dbInput.setFocus()
        elif not self.userInput.text():
            self.userInput.setFocus()
        else:
            self.passInput.setFocus()

        if message:
            self.errorLabel = QLabel(_(message))
            palette = QPalette()
            palette.setColor(QPalette.Foreground, Qt.red)
            self.errorLabel.setPalette(palette)

        inputBox = QVBoxLayout()
        if message:
            inputBox.addWidget(self.errorLabel)
        inputBox.addWidget(self.serverInput)
        inputBox.addWidget(self.dbInput)
        inputBox.addWidget(self.userInput)
        inputBox.addWidget(self.passInput)
        inputBox.addWidget(self.autoLogin)

        winLayout = QVBoxLayout()
        winLayout.addLayout(inputBox)
        winLayout.addLayout(buttonBox)

        self.setLayout(winLayout)
        self.setWindowTitle(_('LoginWindow.title'))
        self.resize(250, 200)
        self._center()

    def _store_results(self):
        self.hide()
        self.deleteLater()
        self._server = self.serverInput.text()
        self._db = self.dbInput.text()
        self._user = self.userInput.text()
        self._pass = self.passInput.text()
        self._ready = True

    def _cancel(self):
        self.hide()
        self.deleteLater()
        self._cancelled = True
        self._ready = True

    def _set_auto(self):
        if self.autoLogin.isChecked():
            put('Login', 'autologin', 'true')
        else:
            put('Login', 'autologin', 'false')

    def _center(self):
        geom = self.frameGeometry()
        centerPoint = QDesktopWidget().availableGeometry().center()
        geom.moveCenter(centerPoint)
        self.move(geom.topLeft())

    def get_info(self):
        self._ready = False
        self.exec_()
        if self._cancelled or not self._ready:
            return (None, None, None, None)

        return (self._user, self._pass, self._server, self._db)

def get_login(message=''):
    l = LoginWindow(message)
    return l.get_info()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Setup script.
"""

import os
import platform
import re
import stat
import sys
import urllib
import zipfile

from bs4 import BeautifulSoup

from settings import JBROWSE_DATA_SYMLINK_PATH
from settings import MEDIA_ROOT
from settings import PWD as GD_ROOT
from settings import TOOLS_DIR

TOOLS_URLS = {
    'bwa' : {
        'Darwin': ['https://www.dropbox.com/s/w6gs82gn0rmc60w/bwa'],
        'Linux': ['https://www.dropbox.com/s/eq3533wx8ay41sx/bwa'],
    },
    'freebayes' : {
        'Darwin' : [
            'https://www.dropbox.com/s/18us45ggflvci6e/bamleftalign',
            'https://www.dropbox.com/s/7pzjie4u4itxyjv/freebayes'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/3qszohowh1gj1u0/bamleftalign',
            'https://www.dropbox.com/s/n2we6c6bmlbyi6f/freebayes'
        ],
    },
    'snpEff' : [
        'https://www.dropbox.com/s/jnob4hkcwtquyag/snpEff-3.3e.zip',
        ],
    'gatk' : [
        #New GATK doesn't work for OSX 10.6 and below b/c it requires Java 7
        #'https://www.dropbox.com/s/47b0ivwlr80vtw1/GenomeAnalysisTK-2.6-5-gba531bd.zip'
        'https://www.dropbox.com/s/227curebr2prswn/GenomeAnalysisTK-legacy.zip'
        ],
    'samtools' : {
        'Linux' : [
            'https://www.dropbox.com/s/pjw0h0vzo2ls79t/samtools-0.1.19-linux.zip'
        ],
        'Darwin' : [
            'https://www.dropbox.com/s/vjtegtyncmq8a8o/samtools-0.1.19-darwin.zip'
        ]
    },
    'tabix' : {
        'Darwin' : [
            'https://www.dropbox.com/s/3mr5x3di2p513ma/tabix-0.2.4-darwin.zip'
        ],
        'Linux': [
            'https://www.dropbox.com/s/38y3p2e7za4conk/tabix-0.2.4-linux.zip'
        ]
    },
    'pindel' : {
        'Darwin' : [
            'https://www.dropbox.com/s/8wuh47i2hao3vlr/pindel-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/wyfve2dfgqbt94d/pindel-linux.zip'
        ]
    },
    'delly' : {
        'Darwin' : [
            'https://www.dropbox.com/s/l3qs0mbm2vapqf8/delly-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/k1gaxm1wb0odubc/delly-linux.zip'
        ]
    },
    'vcf-concat' : {
        'Darwin' : [
            'https://www.dropbox.com/s/ehowcdaic4tln0n/vcf-concat-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/buf80h0qfwjqaqp/vcf-concat-linux.zip'
        ]
    },
    'picard' : [
        'https://www.dropbox.com/s/189loxsmsch1xbb/picard-tools-1.96.zip'
    ]

}

# For any tools that have multiple executables that need permission changes,
# for tools whose executables arent named after their tool
TOOLS_TO_EXECUTABLES = {
    'tabix' : ['tabix','bgzip'],
    'pindel' : ['pindel','pindel2vcf']
}

def setup(arglist):
    if len(arglist) == 0:
        download_tools()
        setup_jbrowse()
    else:
        if 'jbrowse' in arglist:
            setup_jbrowse()
            arglist.remove('jbrowse')
        download_tools(tools=arglist)

def download_tools(tools= TOOLS_URLS.keys()):
    # Create tools dir if it doesn't exist.
    if not os.path.isdir(TOOLS_DIR):
        os.mkdir(TOOLS_DIR)

    sys_type = platform.system()

    # Download the required tools from our Dropbox.
    for tool in tools:

        print 'Grabbing %s from Dropbox...' % tool

        # If the tool is a dict, then it references different OSes (i.e.
        # compiled binaries). Download the correct one for your OS.
        if isinstance(TOOLS_URLS[tool],dict):
            try:
                tool_urls = TOOLS_URLS[tool][sys_type]
            except KeyError, e:
                raise(OSError(' '.join([
                    'Tool "%s" missing for your OS (%s),'
                    'OSs available are: (%s)']) % (
                        tool, sys_type, str(TOOLS_URLS[tool].keys()))))
        else:
            tool_urls = TOOLS_URLS[tool]

        # Make the directory for this tool, if it doesn't exist
        dest_dir = os.path.join(TOOLS_DIR,tool)

        if not os.path.isdir(dest_dir):
            os.mkdir(dest_dir)

        for tool_url in tool_urls:

            # Grab last part of url after final '/' as file name
            dest_filename = re.match(r'^.*\/([^/]+)$', tool_url).group(1)
            dest_path = os.path.join(dest_dir,dest_filename)

            # Get the *actual* file URL from the dropbox popup
            tool_url = _get_file_url_from_dropbox(tool_url, dest_filename)

            # If *.zip, download to a tmp file and unzip it to the right dir.
            if dest_filename.endswith('.zip'):
                tmp_path, http_msg = urllib.urlretrieve(tool_url)
                print '    %s (Unzipping...)' % tmp_path
                tool_zipped = zipfile.ZipFile(tmp_path)
                tool_zipped.extractall(dest_dir)

            # Otherwise download files to the correct location.
            else:
                urllib.urlretrieve(tool_url,dest_path)
                print '    %s' % dest_path

        # This is really hackish, but because ZipFile does not save file
        # permissions, we need to make the bins executable. In cases where
        # the executable name is different than the tool key in TOOL_URLS,
        # use TOOLS_TO_EXECUTABLES.
        if not tool in TOOLS_TO_EXECUTABLES:
            tool_bin_files = [os.path.join(dest_dir,tool)]
        else:
            tool_bin_files = [os.path.join(dest_dir, exe)
                    for exe in TOOLS_TO_EXECUTABLES[tool]]

        for tool_bin_file in tool_bin_files:
            if os.path.exists(tool_bin_file):
                os.chmod(tool_bin_file,
                        stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR)

def _get_file_url_from_dropbox(dropbox_url, filename):
    """
    Dropbox takes you to a download page when you share a link, so to get
    past that, we use BeautifulSoup to get the file, as per goo.gl/EeOGjC
    """
    try:
        url_text = urllib.urlopen(dropbox_url).read()
        url_soup = BeautifulSoup(url_text)
    except:
        raise Exception(
            'Could not get file %s from Dropbox URL %s. Check settings.' % (
                filename, dropbox_url))

    return url_soup.find_all(href=re.compile(filename))[0]['href']

def setup_jbrowse():
    """Sets up JBrowse.
    """
    # Unlink if there was a link and then create a new link.
    try:
        os.unlink(JBROWSE_DATA_SYMLINK_PATH)
    except OSError:
        # There was no symlink. That's fine.
        pass

    # Re-create the link.
    os.symlink(os.path.join(GD_ROOT, MEDIA_ROOT),
            JBROWSE_DATA_SYMLINK_PATH)

if __name__ == '__main__':
    setup(sys.argv[1:])
<commit_msg>Added lumpy to the tools dir.<commit_after>#!/usr/bin/env python

"""
Setup script.
"""

import os
import platform
import re
import stat
import sys
import urllib
import zipfile

from bs4 import BeautifulSoup

from settings import JBROWSE_DATA_SYMLINK_PATH
from settings import MEDIA_ROOT
from settings import PWD as GD_ROOT
from settings import TOOLS_DIR

TOOLS_URLS = {
    'bwa' : {
        'Darwin': ['https://www.dropbox.com/s/w6gs82gn0rmc60w/bwa'],
        'Linux': ['https://www.dropbox.com/s/eq3533wx8ay41sx/bwa'],
    },
    'freebayes' : {
        'Darwin' : [
            'https://www.dropbox.com/s/18us45ggflvci6e/bamleftalign',
            'https://www.dropbox.com/s/7pzjie4u4itxyjv/freebayes'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/3qszohowh1gj1u0/bamleftalign',
            'https://www.dropbox.com/s/n2we6c6bmlbyi6f/freebayes'
        ],
    },
    'snpEff' : [
        'https://www.dropbox.com/s/jnob4hkcwtquyag/snpEff-3.3e.zip',
        ],
    'gatk' : [
        #New GATK doesn't work for OSX 10.6 and below b/c it requires Java 7
        #'https://www.dropbox.com/s/47b0ivwlr80vtw1/GenomeAnalysisTK-2.6-5-gba531bd.zip'
        'https://www.dropbox.com/s/227curebr2prswn/GenomeAnalysisTK-legacy.zip'
        ],
    'samtools' : {
        'Linux' : [
            'https://www.dropbox.com/s/pjw0h0vzo2ls79t/samtools-0.1.19-linux.zip'
        ],
        'Darwin' : [
            'https://www.dropbox.com/s/vjtegtyncmq8a8o/samtools-0.1.19-darwin.zip'
        ]
    },
    'tabix' : {
        'Darwin' : [
            'https://www.dropbox.com/s/3mr5x3di2p513ma/tabix-0.2.4-darwin.zip'
        ],
        'Linux': [
            'https://www.dropbox.com/s/38y3p2e7za4conk/tabix-0.2.4-linux.zip'
        ]
    },
    'pindel' : {
        'Darwin' : [
            'https://www.dropbox.com/s/8wuh47i2hao3vlr/pindel-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/wyfve2dfgqbt94d/pindel-linux.zip'
        ]
    },
    'delly' : {
        'Darwin' : [
            'https://www.dropbox.com/s/l3qs0mbm2vapqf8/delly-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/k1gaxm1wb0odubc/delly-linux.zip'
        ]
    },
    'vcf-concat' : {
        'Darwin' : [
            'https://www.dropbox.com/s/ehowcdaic4tln0n/vcf-concat-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/buf80h0qfwjqaqp/vcf-concat-linux.zip'
        ]
    },
    'picard' : [
        'https://www.dropbox.com/s/189loxsmsch1xbb/picard-tools-1.96.zip'
    ],
    'lumpy' : {
        'Darwin' : [
            'https://www.dropbox.com/s/kzo9tk9lxieaask/lumpy-0.2.1-darwin.zip'
        ],
        'Linux' : [
            'https://www.dropbox.com/s/spivtdr4ud76ba2/lumpy-0.2.1-linux.zip'
        ]
}

# For any tools that have multiple executables that need permission changes,
# for tools whose executables arent named after their tool
TOOLS_TO_EXECUTABLES = {
    'tabix' : ['tabix','bgzip'],
    'pindel' : ['pindel','pindel2vcf']
}

def setup(arglist):
    if len(arglist) == 0:
        download_tools()
        setup_jbrowse()
    else:
        if 'jbrowse' in arglist:
            setup_jbrowse()
            arglist.remove('jbrowse')
        download_tools(tools=arglist)

def download_tools(tools= TOOLS_URLS.keys()):
    # Create tools dir if it doesn't exist.
    if not os.path.isdir(TOOLS_DIR):
        os.mkdir(TOOLS_DIR)

    sys_type = platform.system()

    # Download the required tools from our Dropbox.
    for tool in tools:

        print 'Grabbing %s from Dropbox...' % tool

        # If the tool is a dict, then it references different OSes (i.e.
        # compiled binaries). Download the correct one for your OS.
        if isinstance(TOOLS_URLS[tool],dict):
            try:
                tool_urls = TOOLS_URLS[tool][sys_type]
            except KeyError, e:
                raise(OSError(' '.join([
                    'Tool "%s" missing for your OS (%s),'
                    'OSs available are: (%s)']) % (
                        tool, sys_type, str(TOOLS_URLS[tool].keys()))))
        else:
            tool_urls = TOOLS_URLS[tool]

        # Make the directory for this tool, if it doesn't exist
        dest_dir = os.path.join(TOOLS_DIR,tool)

        if not os.path.isdir(dest_dir):
            os.mkdir(dest_dir)

        for tool_url in tool_urls:

            # Grab last part of url after final '/' as file name
            dest_filename = re.match(r'^.*\/([^/]+)$', tool_url).group(1)
            dest_path = os.path.join(dest_dir,dest_filename)

            # Get the *actual* file URL from the dropbox popup
            tool_url = _get_file_url_from_dropbox(tool_url, dest_filename)

            # If *.zip, download to a tmp file and unzip it to the right dir.
            if dest_filename.endswith('.zip'):
                tmp_path, http_msg = urllib.urlretrieve(tool_url)
                print '    %s (Unzipping...)' % tmp_path
                tool_zipped = zipfile.ZipFile(tmp_path)
                tool_zipped.extractall(dest_dir)

            # Otherwise download files to the correct location.
            else:
                urllib.urlretrieve(tool_url,dest_path)
                print '    %s' % dest_path

        # This is really hackish, but because ZipFile does not save file
        # permissions, we need to make the bins executable. In cases where
        # the executable name is different than the tool key in TOOL_URLS,
        # use TOOLS_TO_EXECUTABLES.
        if not tool in TOOLS_TO_EXECUTABLES:
            tool_bin_files = [os.path.join(dest_dir,tool)]
        else:
            tool_bin_files = [os.path.join(dest_dir, exe)
                    for exe in TOOLS_TO_EXECUTABLES[tool]]

        for tool_bin_file in tool_bin_files:
            if os.path.exists(tool_bin_file):
                os.chmod(tool_bin_file,
                        stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR)

def _get_file_url_from_dropbox(dropbox_url, filename):
    """
    Dropbox takes you to a download page when you share a link, so to get
    past that, we use BeautifulSoup to get the file, as per goo.gl/EeOGjC
    """
    try:
        url_text = urllib.urlopen(dropbox_url).read()
        url_soup = BeautifulSoup(url_text)
    except:
        raise Exception(
            'Could not get file %s from Dropbox URL %s. Check settings.' % (
                filename, dropbox_url))

    return url_soup.find_all(href=re.compile(filename))[0]['href']

def setup_jbrowse():
    """Sets up JBrowse.
    """
    # Unlink if there was a link and then create a new link.
    try:
        os.unlink(JBROWSE_DATA_SYMLINK_PATH)
    except OSError:
        # There was no symlink. That's fine.
        pass

    # Re-create the link.
    os.symlink(os.path.join(GD_ROOT, MEDIA_ROOT),
            JBROWSE_DATA_SYMLINK_PATH)

if __name__ == '__main__':
    setup(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python

#
# Build shell commands from package specifications in a yml database.
#

import getopt
import yaml
import sys
import shlex

def usage():
    print("""
{0} [--pkg=<pkg>] [--pkgs=<pkg1,pkg2>,...] [--script] \
[--docker] [--vagrant] [--split=...] [--distrib=...] \
    /path/to/<example>.yml""".format(sys.argv[0]))

class OutputMode:
    Script, Docker, Vagrant = range(3)

output_mode_spec = dict()
output_mode_spec['script'] = OutputMode.Script
output_mode_spec['docker'] = OutputMode.Docker
output_mode_spec['vagrant'] = OutputMode.Vagrant

def wildcard(spec):
    if 'wildcard' in spec:
        return spec['wildcard']
    else:
        return 'any'

def is_list(a):
    return isinstance(a, list)

def is_dict(a):
    return isinstance(a, dict)

def is_atom(a):
    return not (hasattr(a, '__iter__'))

def get_entry(spec=None, distrib=None, distrib_version=None, pkg=None,
              section=None):
    """
    Get one entry with precedence distrib with version > distrib >
    match distrib > wildcard.
    """

    distrib_full = '{0}-{1}'.format(distrib, distrib_version)

    if pkg in spec[section]:

        if distrib_full in spec[section][pkg]:
            return spec[section][pkg][distrib_full]

        elif distrib in spec[section][pkg]:
            return spec[section][pkg][distrib]

        elif distrib in spec['match']:
            match_distrib = spec['match'][distrib]
            if match_distrib in spec[section][pkg]:
                return spec[section][pkg][match_distrib]

        if wildcard(spec) in spec[section][pkg]:
            return spec[section][pkg][wildcard(spec)]

        else:
            return None

def pkg_entries(spec=None, distrib=None, distrib_version=None, pkg=None):
    """
    Find recursively entries for pkg and distribution distrib in a
    specification spec.
    """

    result = None

    if pkg in spec['pkgs']:
        result = get_entry(spec, distrib, distrib_version, pkg, 'pkgs')

    if result is None:
        return [result]

    elif is_dict(result):
        return [result]
    else:

        if is_atom(result):
            result = [result]
        r = list()

        for e in result:

            if e != pkg:
                ne = pkg_entries(spec=spec, distrib=distrib,
                                 distrib_version=distrib_version, pkg=e)

                if ne == [None]:
                    r.append(e)

                else:
                    r += ne

        return r

def begin(distrib=None, distrib_version=None, output_mode=None):
    """
    Distribution preamble.
    """
    if output_mode == OutputMode.Docker:
        sys.stdout.write('FROM {0}:{1}\n'.format(distrib, distrib_version))

def env(definitions=None, output_mode=None):
    """
    Environment specification.
    """

    if len(definitions) > 0:

        items = list()

        if output_mode == OutputMode.Docker:
            items.append('ENV')

        items += definitions

        sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))

def install(installer=None, command=None, pkg=None, pkgs=None,
            output_mode=OutputMode.Script):
    """
    Format an install command according to output mode.
    """

    if output_mode == OutputMode.Docker:
        items = ['RUN']

    else:
        if output_mode == OutputMode.Script:
            items = []

        else:
            sys.stderr.write('output mode {0} is not implemented\n'.format(
                output_mode))
            exit(1)

    if installer is not None:
        items.append(installer)

    if command is not None:
        if '&&' in command:
            coms = command.split('&&')
            items += ['{0} &&'.format(c.lstrip().rstrip()) for c in coms[:-1]]
            items.append(coms[-1].lstrip().rstrip())
        else:
            items.append(command)

    if pkg is not None:
        items.append(pkg)

    if pkgs is not None:
        items += pkgs

    sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))

class Options(object):

    def __init__(self, *args, **kwargs):
        if len(args) > 0:
            for arg in args:
                self.set_options(arg)

        for k in kwargs:
            setattr(self, k, kwargs[k])

    def set_options(self, values):
        for k in values:
            setattr(self, k, values[k])

    def all(self):
        options = (name for name in dir(self) if not name.startswith('_'))
        values = (self.o for o in options)
        return dict(zip(options, values))

def get_options():

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], '',
                                       ['pkg=',
                                        'pkgs=',
                                        'script',
                                        'docker',
                                        'vagrant',
                                        'split=',
                                        'distrib='])

    except getopt.GetoptError as err:
            sys.stderr.write('{0}\n'.format(str(err)))
            usage()
            exit(2)

    options = Options()

    options.distrib = None
    options.distrib_version = None
    options.pkgs = list()

    options.output_mode = OutputMode.Script
    options.split = False

    for o, a in opts:
        if o == '--distrib':
            if ':' in a:
                options.distrib, options.distrib_version = a.split(':')
            else:
                options.distrib = a
        elif o == '--pkg':
            options.pkgs.append(a)
        elif o == '--pkgs':
            options.pkgs += a.split(',')
        elif o == '--script':
            options.output_mode = OutputMode.Script
        elif o == '--docker':
            options.output_mode = OutputMode.Docker
        elif o == '--vagrant':
            options.output_mode = OutputMode.Vagrant
        elif o == '--split':
            options.split = a.lower() in ['true', 'yes', '1']

    options.specfilename = args[0]

    return options

def print_commands(*args, **kwargs):

    if len(args) == 1:
        options = args[0]
    else:
        options = Options(kwargs)

    with open(options.specfilename) as specfile:

        spec = yaml.load(specfile.read())

        by_installer = list()
        by_command = list()
        definitions = list()

        for pkg in options.pkgs:

            definition = get_entry(spec, options.distrib,
                                   options.distrib_version, pkg, 'env')

            if definition is not None:
                if is_list(definition):
                    for iter_def in definition:
                        definitions.append(iter_def)
                else:
                    definitions.append(definition)

            entries = pkg_entries(spec=spec, distrib=options.distrib,
                                  distrib_version=options.distrib_version,
                                  pkg=pkg)

            for entry in entries:
                if entry is not None:
                    if hasattr(entry, 'has_key'):
                        if 'command' in entry:
                            by_command.append(entry['command'])
                    elif hasattr(entry, 'sort'):
                        by_installer += entry
                    else:
                        by_installer.append(entry)
                else:
                    by_installer.append(pkg)

        begin(distrib=options.distrib, distrib_version=options.distrib_version,
              output_mode=options.output_mode)

        installer = get_entry(spec, options.distrib, options.distrib_version,
                              wildcard(spec), 'installer')

        assert installer is not None

        updater = get_entry(spec, options.distrib, options.distrib_version,
                            wildcard(spec), 'updater')

        if updater:
            installer = '{0} && {1}'.format(updater, installer)

        if options.split:
            for pkg in by_installer:
                format(installer=installer,
                       pkg=pkg, output_mode=options.output_mode)
        else:
            install(installer=installer,
                    pkgs=by_installer, output_mode=options.output_mode)

        for command in by_command:
            install(command=command, output_mode=options.output_mode)

        env(definitions, options.output_mode)

if __name__ == '__main__':
    print_commands(get_options())
<commit_msg>[CI] mksenv.py: add distribution + version in a comment for script output<commit_after>#!/usr/bin/env python

#
# Build shell commands from package specifications in a yml database.
#

import getopt
import yaml
import sys
import shlex

def usage():
    print("""
{0} [--pkg=<pkg>] [--pkgs=<pkg1,pkg2>,...] [--script] \
[--docker] [--vagrant] [--split=...] [--distrib=...] \
    /path/to/<example>.yml""".format(sys.argv[0]))

class OutputMode:
    Script, Docker, Vagrant = range(3)

output_mode_spec = dict()
output_mode_spec['script'] = OutputMode.Script
output_mode_spec['docker'] = OutputMode.Docker
output_mode_spec['vagrant'] = OutputMode.Vagrant

def wildcard(spec):
    if 'wildcard' in spec:
        return spec['wildcard']
    else:
        return 'any'

def is_list(a):
    return isinstance(a, list)

def is_dict(a):
    return isinstance(a, dict)

def is_atom(a):
    return not (hasattr(a, '__iter__'))

def get_entry(spec=None, distrib=None, distrib_version=None, pkg=None,
              section=None):
    """
    Get one entry with precedence distrib with version > distrib >
    match distrib > wildcard.
    """

    distrib_full = '{0}-{1}'.format(distrib, distrib_version)

    if pkg in spec[section]:

        if distrib_full in spec[section][pkg]:
            return spec[section][pkg][distrib_full]

        elif distrib in spec[section][pkg]:
            return spec[section][pkg][distrib]

        elif distrib in spec['match']:
            match_distrib = spec['match'][distrib]
            if match_distrib in spec[section][pkg]:
                return spec[section][pkg][match_distrib]

        if wildcard(spec) in spec[section][pkg]:
            return spec[section][pkg][wildcard(spec)]

        else:
            return None

def pkg_entries(spec=None, distrib=None, distrib_version=None, pkg=None):
    """
    Find recursively entries for pkg and distribution distrib in a
    specification spec.
    """

    result = None

    if pkg in spec['pkgs']:
        result = get_entry(spec, distrib, distrib_version, pkg, 'pkgs')

    if result is None:
        return [result]

    elif is_dict(result):
        return [result]
    else:

        if is_atom(result):
            result = [result]
        r = list()

        for e in result:

            if e != pkg:
                ne = pkg_entries(spec=spec, distrib=distrib,
                                 distrib_version=distrib_version, pkg=e)

                if ne == [None]:
                    r.append(e)

                else:
                    r += ne

        return r

def begin(distrib=None, distrib_version=None, output_mode=None):
    """
    Distribution preamble.
    """
    if output_mode == OutputMode.Docker:
        sys.stdout.write('FROM {0}:{1}\n'.format(distrib, distrib_version))
    elif output_mode == OutputMode.Script:
        sys.stdout.write('# {0} {1}\n'.format(distrib,
                                              distrib_version))

def env(definitions=None, output_mode=None):
    """
    Environment specification.
    """

    if len(definitions) > 0:

        items = list()

        if output_mode == OutputMode.Docker:
            items.append('ENV')

        items += definitions

        sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))

def install(installer=None, command=None, pkg=None, pkgs=None,
            output_mode=OutputMode.Script):
    """
    Format an install command according to output mode.
    """

    if output_mode == OutputMode.Docker:
        items = ['RUN']

    else:
        if output_mode == OutputMode.Script:
            items = []

        else:
            sys.stderr.write('output mode {0} is not implemented\n'.format(
                output_mode))
            exit(1)

    if installer is not None:
        items.append(installer)

    if command is not None:
        if '&&' in command:
            coms = command.split('&&')
            items += ['{0} &&'.format(c.lstrip().rstrip()) for c in coms[:-1]]
            items.append(coms[-1].lstrip().rstrip())
        else:
            items.append(command)

    if pkg is not None:
        items.append(pkg)

    if pkgs is not None:
        items += pkgs

    sys.stdout.write('{0}\n'.format(' \\ \n  '.join(items)))

class Options(object):

    def __init__(self, *args, **kwargs):
        if len(args) > 0:
            for arg in args:
                self.set_options(arg)

        for k in kwargs:
            setattr(self, k, kwargs[k])

    def set_options(self, values):
        for k in values:
            setattr(self, k, values[k])

    def all(self):
        options = (name for name in dir(self) if not name.startswith('_'))
        values = (self.o for o in options)
        return dict(zip(options, values))

def get_options():

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], '',
                                       ['pkg=',
                                        'pkgs=',
                                        'script',
                                        'docker',
                                        'vagrant',
                                        'split=',
                                        'distrib='])

    except getopt.GetoptError as err:
            sys.stderr.write('{0}\n'.format(str(err)))
            usage()
            exit(2)

    options = Options()

    options.distrib = None
    options.distrib_version = None
    options.pkgs = list()

    options.output_mode = OutputMode.Script
    options.split = False

    for o, a in opts:
        if o == '--distrib':
            if ':' in a:
                options.distrib, options.distrib_version = a.split(':')
            else:
                options.distrib = a
        elif o == '--pkg':
            options.pkgs.append(a)
        elif o == '--pkgs':
            options.pkgs += a.split(',')
        elif o == '--script':
            options.output_mode = OutputMode.Script
        elif o == '--docker':
            options.output_mode = OutputMode.Docker
        elif o == '--vagrant':
            options.output_mode = OutputMode.Vagrant
        elif o == '--split':
            options.split = a.lower() in ['true', 'yes', '1']

    options.specfilename = args[0]

    return options

def print_commands(*args, **kwargs):

    if len(args) == 1:
        options = args[0]
    else:
        options = Options(kwargs)

    with open(options.specfilename) as specfile:

        spec = yaml.load(specfile.read())

        by_installer = list()
        by_command = list()
        definitions = list()

        for pkg in options.pkgs:

            definition = get_entry(spec, options.distrib,
                                   options.distrib_version, pkg, 'env')

            if definition is not None:
                if is_list(definition):
                    for iter_def in definition:
                        definitions.append(iter_def)
                else:
                    definitions.append(definition)

            entries = pkg_entries(spec=spec, distrib=options.distrib,
                                  distrib_version=options.distrib_version,
                                  pkg=pkg)

            for entry in entries:
                if entry is not None:
                    if hasattr(entry, 'has_key'):
                        if 'command' in entry:
                            by_command.append(entry['command'])
                    elif hasattr(entry, 'sort'):
                        by_installer += entry
                    else:
                        by_installer.append(entry)
                else:
                    by_installer.append(pkg)

        begin(distrib=options.distrib, distrib_version=options.distrib_version,
              output_mode=options.output_mode)

        installer = get_entry(spec, options.distrib, options.distrib_version,
                              wildcard(spec), 'installer')

        assert installer is not None

        updater = get_entry(spec, options.distrib, options.distrib_version,
                            wildcard(spec), 'updater')

        if updater:
            installer = '{0} && {1}'.format(updater, installer)

        if options.split:
            for pkg in by_installer:
                format(installer=installer,
                       pkg=pkg, output_mode=options.output_mode)
        else:
            install(installer=installer,
                    pkgs=by_installer, output_mode=options.output_mode)

        for command in by_command:
            install(command=command, output_mode=options.output_mode)

        env(definitions, options.output_mode)

if __name__ == '__main__':
    print_commands(get_options())
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding:Utf-8 -*-

#  Dieren Theater - lachambre.be to json sausage machine
#  Copyright (C) 2011  Laurent Peuch <cortex@worlddomination.be>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
from json import dumps
from json_diff import Comparator
from StringIO import StringIO

SEPARATOR = "-------------------------------------------------------------------------------------------------------------------------------"

def pdf_to_text(pdf_name):
    os.system("pdftotext -layout %s" % pdf_name)
    return open(pdf_name.replace(".pdf", ".txt"), "r").read()

def remove_useless_informations(text):
    result = []
    for i in text.split("\n")[1:]:
        if "DOC 53" in i:
            continue
        if "\x0c" in i:
            continue
        if re.match("^CHAMBRE .* ZITTINGSPERIODE$", i):
            continue
        if re.match("^ *\d\d\d\d *$", i):
            continue
        if "Imprimerie centrale – Cette publication est imprimée exclusivement sur du papier certiﬁé FSC" in i:
            continue
        if "Centrale drukkerij – Deze publicatie wordt uitsluitend gedrukt op FSC gecertiﬁceerd papier" in i:
            continue
        if "Publications ofﬁcielles éditées par la Chambre des représentants" in i:
            continue
        else:
            result.append(i)
    return "\n".join(result)

def strip(text):
    work = text.split("\n")
    a = 0
    while re.match("^ *$", work[a]):
        a += 1

    b = 0
    while re.match("^ *$", work[b]):
        b += 1

    return "\n".join(text.split("\n")[a:-b])

def horizontal_split(text):
    result = [[]]
    blank_line = 0
    for i in text.split("\n"):
        if i.strip():
            result[-1].append(i)
        else:
            blank_line += 1
            if blank_line == 3:
                result.append([])
                blank_line = 0
    return filter(None, result)

def remove_useless_blocks(text):
    # first one is "La Chambre blablabla"
    # second one is the date, we have it on the website
    # last one is the authors, also on the website
    text = text[2:-1]
    # author name, also on the website
    if "SAMENVATTING" not in text[1][0]:
        text.pop(1)
    return text

def split_horizontally(block):
    left, right = [], []

    if not block:
        return left, right

    split_size = 2

    searchable_area = filter(lambda x: len(filter(lambda x: x.strip(), x.split("   "))) == 2, block[:])
    searchable_area = map(lambda x: x.decode("Utf-8"), searchable_area)
    split_index = 7
    good = False
    while not good:
        for i in searchable_area:
            if i[split_index:split_index+2] != "  ":
                split_index += 1
                break
        else:
            good = True
        if split_index > max(map(lambda x: len(x), searchable_area)):
            raise Exception

    for i in map(lambda x: x.decode("Utf-8"), block):
        if len(filter(lambda x: x.strip(), i.split(" "*split_size))) == 2:
            left.append(i[:split_index])
            right.append(i[split_index+2:])
        else:
            left.append(i.rstrip())

    return map(lambda x: x.encode("Utf-8"), left), map(lambda x: x.encode("Utf-8"), right)

def rebuild_paragraphe(block):
    # '-' are at the end of a line when a word is split in two
    # after " ".join we have some "- " resulting of the join of 2 parts of a word
    # remove them
    return " ".join(map(lambda x: x.strip(), block)).replace("- ", "")

def parse_abstract(abstract):
    # first line is useless
    abstract[0] = abstract[0].lower().strip()
    return map(lambda x: x.capitalize() + ".", map(rebuild_paragraphe, split_horizontally(abstract)))

def left_egualize(block):
    # if I got empty text just strip it
    if not filter(lambda y: filter(lambda x: x.strip(), y), block):
        return map(lambda x: x.strip(), block)

    while False not in map(lambda x: x[0] == " ", filter(lambda x: x.strip(), block)):
        block = map(lambda x: x[1:], block)
    return block

def split_raw_paragraph(block):
    block = left_egualize(block)
    result = [[]]
    for i in block:
        if i.startswith("  ") and result != [[]]:
            result.append([])
        result[-1].append(i)
    return result

def parse_two_columns_text(text):
    return map(lambda x: map(rebuild_paragraphe, x), map(split_raw_paragraph, split_horizontally(text)))

def prepare_document(pdf_name):
    text = pdf_to_text(pdf_name)
    text = remove_useless_informations(text)
    text = strip(text)
    text = horizontal_split(text)
    text = remove_useless_blocks(text)
    return text

def is_party_list(block):
    for i in block:
        if ":" not in i:
            return False
    return True

def flaten_list(a_list):
    flat = []
    for i in a_list:
        for j in i:
            flat.append(j)
    return flat

def intelligent_parse(pdf_name):
    def store(key, data):
        document["FR"][key] = data[0]
        document["NL"][key] = data[1]

    text = prepare_document(pdf_name)
    document = {"FR": {}, "NL": {}}

    # abstract is always the first one
    abstract = parse_abstract(text[0])
    store("abstract", abstract)

    # remove abstract
    text.pop(0)

    # if we have a summary
    if "SAMENVATTING" in text[0][0]:
        text.pop(0)
        summary = map(rebuild_paragraphe, split_horizontally(text[0]))
        store("summary", summary)
        text.pop(0)

    if is_party_list(text[0]):
        # if it's the party list, we will have the abbreviations and address also
        # and we don't care about those
        text = text[3:]

    if "TOELICHTING" in text[0][0]:
        text.pop(0)
        development = []
        while "PROPOSITION DE LOI" not in text[0][0] and "COMMENTAIRE DES ARTICLES" not in text[0][0]:
            development.append(text.pop(0))

        # remove author signature
        development.pop()
        store("development", map(flaten_list, zip(*map(parse_two_columns_text, development))))

    if "COMMENTAIRE DES ARTICLES" in text[0][0]:
        text.pop(0)

        comments = []
        while "PROPOSITION DE LOI" not in text[0][0]:
            comments.append(text.pop(0))
        comments.pop()
        store("comments", map(flaten_list, zip(*map(parse_two_columns_text, comments))))

    if "PROPOSITION DE LOI" in text[0][0]:
        text.pop(0)
        store("articles", map(flaten_list, zip(*map(parse_two_columns_text, text))))

    return document

def custom_parse(pdf_name):
    text = prepare_document(pdf_name)
    for i in parse_abstract(text[0]):
        print SEPARATOR
        print i
    # text[1] is "RESUMER ..."
    for i in map(rebuild_paragraphe, split_horizontally(text[2])):
        print SEPARATOR
        print i
    # text[3] is party list
    # text[4] is abbrevations
    # text[5] is la chambre's address
    # text[6] is "DÉVELOPPEMENTS ..."
    for x in text[7:14]:
        print SEPARATOR
        for i in parse_two_columns_text(x):
            for j in i:
                print SEPARATOR
                print j
    # text[15] is the authors
    # text[16] is "PROPOSITION DE LOI ..."
    for i in parse_two_columns_text(text[17]):
        for j in i:
            print SEPARATOR
            print j
    for i in parse_two_columns_text(text[18]):
        for j in i:
            print SEPARATOR
            print j

if __name__ == "__main__":
    #custom_parse("53K1961001.pdf")
    assert not Comparator(StringIO(dumps(intelligent_parse("53K1961001.pdf"))), open("testing", "r")).compare_dicts()
<commit_msg>[enh] add verification code to avoid regressions<commit_after>#!/usr/bin/python
# -*- coding:Utf-8 -*-

#  Dieren Theater - lachambre.be to json sausage machine
#  Copyright (C) 2011  Laurent Peuch <cortex@worlddomination.be>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as
#  published by the Free Software Foundation, either version 3 of the
#  License, or any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import re
from json import dumps
from json_diff import Comparator
from StringIO import StringIO

SEPARATOR = "-------------------------------------------------------------------------------------------------------------------------------"

def pdf_to_text(pdf_name):
    os.system("pdftotext -layout %s" % pdf_name)
    return open(pdf_name.replace(".pdf", ".txt"), "r").read()

def remove_useless_informations(text):
    result = []
    for i in text.split("\n")[1:]:
        if "DOC 53" in i:
            continue
        if "\x0c" in i:
            continue
        if re.match("^CHAMBRE .* ZITTINGSPERIODE$", i):
            continue
        if re.match("^ *\d\d\d\d *$", i):
            continue
        if "Imprimerie centrale – Cette publication est imprimée exclusivement sur du papier certiﬁé FSC" in i:
            continue
        if "Centrale drukkerij – Deze publicatie wordt uitsluitend gedrukt op FSC gecertiﬁceerd papier" in i:
            continue
        if "Publications ofﬁcielles éditées par la Chambre des représentants" in i:
            continue
        else:
            result.append(i)
    return "\n".join(result)

def strip(text):
    work = text.split("\n")
    a = 0
    while re.match("^ *$", work[a]):
        a += 1

    b = 0
    while re.match("^ *$", work[b]):
        b += 1

    return "\n".join(text.split("\n")[a:-b])

def horizontal_split(text):
    result = [[]]
    blank_line = 0
    for i in text.split("\n"):
        if i.strip():
            result[-1].append(i)
        else:
            blank_line += 1
            if blank_line == 3:
                result.append([])
                blank_line = 0
    return filter(None, result)

def remove_useless_blocks(text):
    # first one is "La Chambre blablabla"
    # second one is the date, we have it on the website
    # last one is the authors, also on the website
    text = text[2:-1]
    # author name, also on the website
    if "SAMENVATTING" not in text[1][0]:
        text.pop(1)
    return text

def split_horizontally(block):
    left, right = [], []

    if not block:
        return left, right

    split_size = 2

    searchable_area = filter(lambda x: len(filter(lambda x: x.strip(), x.split("   "))) == 2, block[:])
    searchable_area = map(lambda x: x.decode("Utf-8"), searchable_area)
    split_index = 7
    good = False
    while not good:
        for i in searchable_area:
            if i[split_index:split_index+2] != "  ":
                split_index += 1
                break
        else:
            good = True
        if split_index > max(map(lambda x: len(x), searchable_area)):
            raise Exception

    for i in map(lambda x: x.decode("Utf-8"), block):
        if len(filter(lambda x: x.strip(), i.split(" "*split_size))) == 2:
            left.append(i[:split_index])
            right.append(i[split_index+2:])
        else:
            left.append(i.rstrip())

    return map(lambda x: x.encode("Utf-8"), left), map(lambda x: x.encode("Utf-8"), right)

def rebuild_paragraphe(block):
    # '-' are at the end of a line when a word is split in two
    # after " ".join we have some "- " resulting of the join of 2 parts of a word
    # remove them
    return " ".join(map(lambda x: x.strip(), block)).replace("- ", "")

def parse_abstract(abstract):
    # first line is useless
    abstract[0] = abstract[0].lower().strip()
    return map(lambda x: x.capitalize() + ".", map(rebuild_paragraphe, split_horizontally(abstract)))

def left_egualize(block):
    # if I got empty text just strip it
    if not filter(lambda y: filter(lambda x: x.strip(), y), block):
        return map(lambda x: x.strip(), block)

    while False not in map(lambda x: x[0] == " ", filter(lambda x: x.strip(), block)):
        block = map(lambda x: x[1:], block)
    return block

def split_raw_paragraph(block):
    block = left_egualize(block)
    result = [[]]
    for i in block:
        if i.startswith("  ") and result != [[]]:
            result.append([])
        result[-1].append(i)
    return result

def parse_two_columns_text(text):
    return map(lambda x: map(rebuild_paragraphe, x), map(split_raw_paragraph, split_horizontally(text)))

def prepare_document(pdf_name):
    text = pdf_to_text(pdf_name)
    text = remove_useless_informations(text)
    text = strip(text)
    text = horizontal_split(text)
    text = remove_useless_blocks(text)
    return text

def is_party_list(block):
    for i in block:
        if ":" not in i:
            return False
    return True

def flaten_list(a_list):
    flat = []
    for i in a_list:
        for j in i:
            flat.append(j)
    return flat

def intelligent_parse(pdf_name):
    def store(key, data):
        document["FR"][key] = data[0]
        document["NL"][key] = data[1]

    text = prepare_document(pdf_name)
    document = {"FR": {}, "NL": {}}

    # abstract is always the first one
    abstract = parse_abstract(text[0])
    store("abstract", abstract)

    # remove abstract
    text.pop(0)

    # if we have a summary
    if "SAMENVATTING" in text[0][0]:
        text.pop(0)
        summary = map(rebuild_paragraphe, split_horizontally(text[0]))
        store("summary", summary)
        text.pop(0)

    if is_party_list(text[0]):
        # if it's the party list, we will have the abbreviations and address also
        # and we don't care about those
        text = text[3:]

    if "TOELICHTING" in text[0][0]:
        text.pop(0)
        development = []
        while "PROPOSITION DE LOI" not in text[0][0] and "COMMENTAIRE DES ARTICLES" not in text[0][0]:
            development.append(text.pop(0))

        # remove author signature
        development.pop()
        store("development", map(flaten_list, zip(*map(parse_two_columns_text, development))))

    if "COMMENTAIRE DES ARTICLES" in text[0][0]:
        text.pop(0)

        comments = []
        while "PROPOSITION DE LOI" not in text[0][0]:
            comments.append(text.pop(0))
        comments.pop()
        store("comments", map(flaten_list, zip(*map(parse_two_columns_text, comments))))

    if "PROPOSITION DE LOI" in text[0][0]:
        text.pop(0)
        store("articles", map(flaten_list, zip(*map(parse_two_columns_text, text))))

    return document

def custom_parse(pdf_name):
    text = prepare_document(pdf_name)
    for i in parse_abstract(text[0]):
        print SEPARATOR
        print i
    # text[1] is "RESUMER ..."
    for i in map(rebuild_paragraphe, split_horizontally(text[2])):
        print SEPARATOR
        print i
    # text[3] is party list
    # text[4] is abbrevations
    # text[5] is la chambre's address
    # text[6] is "DÉVELOPPEMENTS ..."
    for x in text[7:14]:
        print SEPARATOR
        for i in parse_two_columns_text(x):
            for j in i:
                print SEPARATOR
                print j
    # text[15] is the authors
    # text[16] is "PROPOSITION DE LOI ..."
    for i in parse_two_columns_text(text[17]):
        for j in i:
            print SEPARATOR
            print j
    for i in parse_two_columns_text(text[18]):
        for j in i:
            print SEPARATOR
            print j

if __name__ == "__main__":
    for i in ("53K1961001", "53K1098001"):
        try:
            assert not Comparator(StringIO(dumps(intelligent_parse("%s.pdf" % i))), open("%s.json" % i, "r")).compare_dicts()
        except AssertionError:
            print Comparator(StringIO(dumps(intelligent_parse("%s.pdf" % i))), open("%s.json" % i, "r")).compare_dicts()
<|endoftext|>


<commit_before>import email
import json
import smtplib

from email.MIMEText import MIMEText

import requests

def json_serialize(obj):
    """handle datetime objects"""

    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    else:
        raise TypeError, 'Object of type %s with value of %s is not JSON \
serializable' % (type(obj), repr(obj))

def cleanup_statuses(statuses):
    """Downcase keys and only include relevant items"""

    def _format_item(item):
        status = {'status': item['Status']}
        status.update({
            e['Name']: e['Status'] for e in item['Details']
        })

        return status

    info = {}

    for status in statuses:
        instance = status['InstanceId']
        status = {
            'events': status.get('Events', []),
            'instance': _format_item(status['InstanceStatus']),
            'system': _format_item(status['SystemStatus']),
        }

        info[instance] = status

    return info

def send(data, config):
    notify_fcns = {
        'json': notify_json,
        'slack': notify_slack,
        'email': notify_email,
    }

    data['instances'] = cleanup_statuses(data['instances'])

    for key, fcn in notify_fcns.iteritems():
        if key in config:
            fcn(data, config[key])

def notify_json(data, config):
    if 'file' in config:
        with open(config['file'], 'w') as fp:
            json.dump(data, fp, indent=4, sort_keys=True, default=json_serialize)
    else:
        print json.dumps(data, indent=4, sort_keys=True, default=json_serialize)

def notify_slack(data, config):
    # Don't spam slack if there's nothing wrong
    if not len(data['instances']):
        return

    instances = ', '.join(data['instances'].keys())
    post_data = {
        'text': '!!! %d instance(s) have an issue: %s' % (
            len(data['instances']), instances)
    }

    # Add in the optional keys
    if 'user_name' in config:
        post_data['username'] = config['user_name']

    if 'channel' in config:
        post_data['channel'] = config['channel']

    res = requests.post(config['hook_url'], json=post_data)
    res.raise_for_status()

def notify_email(data, config):

    msg_content = '''
    {num_instances} EC2 instances currently have issues!
    ========

    By AWS profile:
    --------

    {profile_counts}

    Full details:
    --------

    {details}
    '''.format(
        num_instances=len(data['instances']),
        profile_counts='\n'.join([
            '- %s: %s' % (name, ', '.join(inst))
            for name, inst in data['profiles']
        ]),
        details=json.dumps(data, indent=4, sort_keys=True, default=json_serialize)
    )

    msg = MIMEText(msg_content, 'plain')
    msg['Subject'] = config['subject']
    msg['From'] = config.get('sender', config['user_name'])
    msg['To'] = ', '.join(config['recipients'])

    smtp = smtplib.SMTP_SSL(config['server'])
    smtp.login(config['user_name'], config['password'])
    smtp.sendmail(config['sender'], config['recipients'], msg.as_string())
    smtp.close()
<commit_msg>don't notify when everything is fine<commit_after>import email
import json
import smtplib

from email.MIMEText import MIMEText

import requests

def json_serialize(obj):
    """handle datetime objects"""

    if hasattr(obj, 'isoformat'):
        return obj.isoformat()
    else:
        raise TypeError, 'Object of type %s with value of %s is not JSON \
serializable' % (type(obj), repr(obj))

def cleanup_statuses(statuses):
    """Downcase keys and only include relevant items"""

    def _format_item(item):
        status = {'status': item['Status']}
        status.update({
            e['Name']: e['Status'] for e in item['Details']
        })

        return status

    info = {}

    for status in statuses:
        instance = status['InstanceId']
        status = {
            'events': status.get('Events', []),
            'instance': _format_item(status['InstanceStatus']),
            'system': _format_item(status['SystemStatus']),
        }

        info[instance] = status

    return info

def send(data, config):
    notify_fcns = {
        'json': notify_json,
        'slack': notify_slack,
        'email': notify_email,
    }

    # Don't notify unless there's actually something wrong
    if not len(data['instances']):
        return

    data['instances'] = cleanup_statuses(data['instances'])

    for key, fcn in notify_fcns.iteritems():
        if key in config:
            fcn(data, config[key])

def notify_json(data, config):
    if 'file' in config:
        with open(config['file'], 'w') as fp:
            json.dump(data, fp, indent=4, sort_keys=True, default=json_serialize)
    else:
        print json.dumps(data, indent=4, sort_keys=True, default=json_serialize)

def notify_slack(data, config):
    instances = ', '.join(data['instances'].keys())
    post_data = {
        'text': '!!! %d instance(s) have an issue: %s' % (
            len(data['instances']), instances)
    }

    # Add in the optional keys
    if 'user_name' in config:
        post_data['username'] = config['user_name']

    if 'channel' in config:
        post_data['channel'] = config['channel']

    res = requests.post(config['hook_url'], json=post_data)
    res.raise_for_status()

def notify_email(data, config):

    msg_content = '''
    {num_instances} EC2 instances currently have issues!
    ========

    By AWS profile:
    --------

    {profile_counts}

    Full details:
    --------

    {details}
    '''.format(
        num_instances=len(data['instances']),
        profile_counts='\n'.join([
            '- %s: %s' % (name, ', '.join(inst))
            for name, inst in data['profiles']
        ]),
        details=json.dumps(data, indent=4, sort_keys=True, default=json_serialize)
    )

    msg = MIMEText(msg_content, 'plain')
    msg['Subject'] = config['subject']
    msg['From'] = config.get('sender', config['user_name'])
    msg['To'] = ', '.join(config['recipients'])

    smtp = smtplib.SMTP_SSL(config['server'])
    smtp.login(config['user_name'], config['password'])
    smtp.sendmail(config['sender'], config['recipients'], msg.as_string())
    smtp.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# stdlib
import tempfile
import time
import os
import shutil
import sys
import uuid

# pypi
import sh
from retask.queue import Queue
from sqlalchemy import engine_from_config
from pyramid.paster import (
    get_appsettings,
    setup_logging,
)

import pep8

# local
import pep8bot.githubutils as gh
import pep8bot.models as m

class directory(object):
    """ pushd/popd context manager. """
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self, *args, **kw):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, *args, **kw):
        os.chdir(self.savedPath)

class Worker(object):
    """ Represents the worker process.  Waits for tasks to come in from the
    webapp and then acts on them.
    """

    def __init__(self, config_uri):
        setup_logging(config_uri)
        settings = get_appsettings(config_uri, name="pep8bot")
        engine = engine_from_config(settings, 'sqlalchemy.')
        m.DBSession.configure(bind=engine)

        # TODO -- pass in redis params from config, hostname, etc.
        self.queue = Queue('commits')
        self.queue.connect()
        # TODO -- set both of these with the config file.
        # Use pyramid tools to load config.
        self.sleep_interval = 1
        self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
        try:
            os.makedirs(self.scratch_dir)
        except OSError:
            pass  # Assume that the scratch_dir already exists.

    def run(self):
        while True:
            time.sleep(self.sleep_interval)
            print "Waking"
            if self.queue.length == 0:
                continue

            task = self.queue.dequeue()
            data = task.data

            repo = data['repository']['name']
            owner = data['repository']['owner']['name']
            commits = data['commits']
            _user = m.User.query.filter_by(username=owner).one()
            token = _user.oauth_access_token

            # We used to fork repos in order to issue pull requests.. but thats
            # not necessary anymore...
            url = data['repository']['url']
            #fork = gh.my_fork(owner, repo)
            #if not fork:
            #    fork = gh.create_fork(owner, repo)
            #    print "Sleeping for 4 seconds"
            #    time.sleep(4)

            #url = fork['ssh_url']

            self.working_dir = tempfile.mkdtemp(
                prefix=owner + '-' + repo,
                dir=self.scratch_dir,
            )

            print "** Cloning to", self.working_dir
            print sh.git.clone(url, self.working_dir)

            print "** Adding remote upstream"
            with directory(self.working_dir):
                print sh.git.remote.add("upstream", data['repository']['url'])
                print sh.git.pull("upstream",
                                  data['repository']['master_branch'])

            lookup = {
                "success": "PEP8bot says \"OK\"",
                "failure": "PEP8bot detected {n} errors",
                "pending": "Still waiting on PEP8bot check",
                "error": "PEP8bot ran into trouble",
            }
            print "** Processing commits."
            for commit in commits:
                sha = commit['id']
                # TODO -- what about hash collisions?
                _commit = m.Commit.query.filter_by(sha=sha).one()
                import transaction
                try:
                    print "** Processing files on commit", sha
                    with directory(self.working_dir):
                        print sh.git.checkout(sha)

                    # TODO -- only process those in modified and added
                    infiles = []
                    for root, dirs, files in os.walk(self.working_dir):

                        if '.git' in root:
                            continue

                        infiles.extend([
                            root + "/" + fname
                            for fname in files
                            if fname.endswith(".py")
                        ])

                    # TODO -- document that the user can keep a .config/pep8
                    # file in their project dir.
                    pep8style = pep8.StyleGuide(
                        quiet=True,
                        config_file="./.config/pep8",
                    )
                    result = pep8style.check_files(infiles)

                    if result.total_errors > 0:
                        status = "failure"
                    else:
                        status = "success"

                    desc = lookup[status].format(n=result.total_errors)
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                except Exception:
                    status = "error"
                    desc = lookup[status]
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                    raise
                finally:
                    transaction.commit()

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def worker(config_filename):
    w = Worker(config_filename)
    try:
        w.run()
    except KeyboardInterrupt:
        pass

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    worker(sys.argv[1])

if __name__ == '__main__':
    main()
<commit_msg>Quiet up that worker process.<commit_after>#!/usr/bin/env python

# stdlib
import tempfile
import time
import os
import shutil
import sys
import uuid

# pypi
import sh
from retask.queue import Queue
from sqlalchemy import engine_from_config
from pyramid.paster import (
    get_appsettings,
    setup_logging,
)

import pep8

# local
import pep8bot.githubutils as gh
import pep8bot.models as m

class directory(object):
    """ pushd/popd context manager. """
    def __init__(self, newPath):
        self.newPath = newPath

    def __enter__(self, *args, **kw):
        self.savedPath = os.getcwd()
        os.chdir(self.newPath)

    def __exit__(self, *args, **kw):
        os.chdir(self.savedPath)

class Worker(object):
    """ Represents the worker process.  Waits for tasks to come in from the
    webapp and then acts on them.
    """

    def __init__(self, config_uri):
        setup_logging(config_uri)
        settings = get_appsettings(config_uri, name="pep8bot")
        engine = engine_from_config(settings, 'sqlalchemy.')
        m.DBSession.configure(bind=engine)

        # TODO -- pass in redis params from config, hostname, etc.
        self.queue = Queue('commits')
        self.queue.connect()
        # TODO -- set both of these with the config file.
        # Use pyramid tools to load config.
        self.sleep_interval = 1
        self.scratch_dir = "/home/threebean/scratch/pep8bot-scratch"
        try:
            os.makedirs(self.scratch_dir)
        except OSError:
            pass  # Assume that the scratch_dir already exists.

    def run(self):
        while True:
            time.sleep(self.sleep_interval)
            task = self.queue.wait()
            data = task.data

            repo = data['repository']['name']
            owner = data['repository']['owner']['name']
            commits = data['commits']
            _user = m.User.query.filter_by(username=owner).one()
            token = _user.oauth_access_token

            # We used to fork repos in order to issue pull requests.. but thats
            # not necessary anymore...
            url = data['repository']['url']
            #fork = gh.my_fork(owner, repo)
            #if not fork:
            #    fork = gh.create_fork(owner, repo)
            #    print "Sleeping for 4 seconds"
            #    time.sleep(4)

            #url = fork['ssh_url']

            self.working_dir = tempfile.mkdtemp(
                prefix=owner + '-' + repo,
                dir=self.scratch_dir,
            )

            print "** Cloning to", self.working_dir
            print sh.git.clone(url, self.working_dir)

            print "** Adding remote upstream"
            with directory(self.working_dir):
                print sh.git.remote.add("upstream", data['repository']['url'])
                print sh.git.pull("upstream",
                                  data['repository']['master_branch'])

            lookup = {
                "success": "PEP8bot says \"OK\"",
                "failure": "PEP8bot detected {n} errors",
                "pending": "Still waiting on PEP8bot check",
                "error": "PEP8bot ran into trouble",
            }
            print "** Processing commits."
            for commit in commits:
                sha = commit['id']
                # TODO -- what about hash collisions?
                _commit = m.Commit.query.filter_by(sha=sha).one()
                import transaction
                try:
                    print "** Processing files on commit", sha
                    with directory(self.working_dir):
                        print sh.git.checkout(sha)

                    # TODO -- only process those in modified and added
                    infiles = []
                    for root, dirs, files in os.walk(self.working_dir):

                        if '.git' in root:
                            continue

                        infiles.extend([
                            root + "/" + fname
                            for fname in files
                            if fname.endswith(".py")
                        ])

                    # TODO -- document that the user can keep a .config/pep8
                    # file in their project dir.
                    pep8style = pep8.StyleGuide(
                        quiet=True,
                        config_file="./.config/pep8",
                    )
                    result = pep8style.check_files(infiles)

                    if result.total_errors > 0:
                        status = "failure"
                    else:
                        status = "success"

                    desc = lookup[status].format(n=result.total_errors)
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                except Exception:
                    status = "error"
                    desc = lookup[status]
                    _commit.status = status
                    gh.post_status(owner, repo, sha, status, token, desc)
                    raise
                finally:
                    transaction.commit()

def usage(argv):
    cmd = os.path.basename(argv[0])
    print('usage: %s <config_uri>\n'
          '(example: "%s development.ini")' % (cmd, cmd))
    sys.exit(1)

def worker(config_filename):
    w = Worker(config_filename)
    try:
        w.run()
    except KeyboardInterrupt:
        pass

def main(argv=sys.argv):
    if len(argv) != 2:
        usage(argv)
    worker(sys.argv[1])

if __name__ == '__main__':
    main()
<|endoftext|>