<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Authors: Muschang Anthony
#    Copyright (c) 2013 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import openerp.tests.common as common
import logging

_logger = logging.getLogger(__name__)

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

class test_add_partners(common.TransactionCase):

    def test_duplicate(self):
        """
            Test adding duplicate partners in an event
        """

        partner_model = self.registry('res.partner')
        event_model = self.registry('event.event')

        user_id = self.ref("base.user_demo")

        #CREATE three partners
        partners_ids = []
        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'John',
        }))

        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Jack',
        }))

        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Carl',
        }))

        #CREATE an event
        event_id = event_model.create(self.cr, user_id, {
            'name': 'test event',
            'date_begin': '01/01/2013',
            'date_end': '02/01/2013',
        })

        #Add the partners in the event
        partners = partner_model.browse(self.cr, self.uid, partners_ids)
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add the partners in the event again
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add the partners with a new one in the event again
        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Bob',
        }))
        partners = partner_model.browse(self.cr, self.uid, partners_ids)
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add 0 partners
        event_model.add_multiple_partner(self.cr, user_id, event_id, [])
        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

    def _check_partner_with_partner_in_event_registration(self, partners_ids, event):
        self.assertEqual(len(event.registration_ids), len(partners_ids),
                         "The registration count should be the same as the number of partners added")

        partners_in_registration = []
        i = 0
        while i < len(partners_ids):
            partners_in_registration.append(event.registration_ids[i].partner_id.id)
            i += 1

        self.assertEqual(set(partners_in_registration), set(partners_ids),
                         "Each registration should correspond with the partners added")

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[FIX]event_multiple_registration: Fix the test<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Authors: Muschang Anthony
#    Copyright (c) 2013 Acsone SA/NV (http://www.acsone.eu)
#    All Rights Reserved
#
#    WARNING: This program as such is intended to be used by professional
#    programmers who take the whole responsibility of assessing all potential
#    consequences resulting from its eventual inadequacies and bugs.
#    End users who are looking for a ready-to-use solution with commercial
#    guarantees and support are strongly advised to contact a Free Software
#    Service Company.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import openerp.tests.common as common
import logging
import datetime

_logger = logging.getLogger(__name__)

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

class test_add_partners(common.TransactionCase):

    def test_duplicate(self):
        """
            Test adding duplicate partners in an event
        """

        partner_model = self.registry('res.partner')
        event_model = self.registry('event.event')

        user_id = self.ref("base.user_demo")

        #CREATE three partners
        partners_ids = []
        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'John',
        }))

        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Jack',
        }))

        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Carl',
        }))

        #CREATE an event
        event_id = event_model.create(self.cr, user_id, {
            'name': 'test event',
            'date_begin': datetime.date(2013, 01, 01),
            'date_end': datetime.date(2013, 02, 01),
        })

        #Add the partners in the event
        partners = partner_model.browse(self.cr, self.uid, partners_ids)
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add the partners in the event again
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add the partners with a new one in the event again
        partners_ids.append(partner_model.create(self.cr, user_id, {
            'name': 'Bob',
        }))
        partners = partner_model.browse(self.cr, self.uid, partners_ids)
        event_model.add_multiple_partner(self.cr, user_id, event_id, partners)

        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

        #Add 0 partners
        event_model.add_multiple_partner(self.cr, user_id, event_id, [])
        event = event_model.browse(self.cr, self.uid, event_id)
        self._check_partner_with_partner_in_event_registration(partners_ids, event)

    def _check_partner_with_partner_in_event_registration(self, partners_ids, event):
        self.assertEqual(len(event.registration_ids), len(partners_ids),
                         "The registration count should be the same as the number of partners added")

        partners_in_registration = []
        i = 0
        while i < len(partners_ids):
            partners_in_registration.append(event.registration_ids[i].partner_id.id)
            i += 1

        self.assertEqual(set(partners_in_registration), set(partners_ids),
                         "Each registration should correspond with the partners added")

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/env python
import asyncio
import os
import sys
import contextlib
from pathlib import Path
from typing import Optional, List, Dict, Generator
import configargparse
from booru_toolkit import errors
from booru_toolkit import cli
from booru_toolkit import util
from booru_toolkit.plugin import PluginBase
from booru_toolkit.plugin import Post
from booru_toolkit.plugin import PluginGelbooru
from booru_toolkit.plugin import PluginPixiv
from booru_toolkit.plugin import PluginYume

PLUGINS: List[PluginBase] = [PluginGelbooru(), PluginPixiv(), PluginYume()]

class DownloadHistory:
    def __init__(self, path: Path) -> None:
        self._path = path
        self._downloaded_urls: Dict[str, bool] = {}
        self._buffered_urls: List[str] = []
        if self._path.exists():
            for line in self._path.open('r'):
                self._downloaded_urls[line.strip()] = True

    def add(self, url: str) -> None:
        if self.is_downloaded(url):
            return
        self._downloaded_urls[url] = True
        self._buffered_urls.append(url)

    def flush(self) -> None:
        self._path.parent.mkdir(parents=True, exist_ok=True)
        with self._path.open('a') as handle:
            for url in self._buffered_urls:
                print(url, file=handle)
        self._buffered_urls = []

    def is_downloaded(self, url: str) -> bool:
        return url in self._downloaded_urls

class DownloadStats:
    def __init__(self) -> None:
        self.errors = 0
        self.ignored = 0
        self.downloaded = 0

    def ignore(self, url: str, reason: str) -> None:
        print('{}: ignored ({})'.format(url, reason))
        self.ignored += 1

    @contextlib.contextmanager
    def download(self, url: str) -> Generator:
        try:
            print('{}: downloading...'.format(url))
            yield
            self.downloaded += 1
            print('{}: saved'.format(url))
        except Exception as ex:
            print('error: {}'.format(ex))
            self.errors += 1
            raise

class Downloader:
    def __init__(
            self,
            plugin: PluginBase,
            target_dir: Path,
            history: Optional[DownloadHistory],
            force: bool,
            max_attempts: int,
            max_concurrent_downloads: int,
            sleep: float) -> None:
        self._plugin = plugin
        self._target_dir = target_dir
        self._history = history
        self._force = force
        self._max_attempts = max_attempts
        self._max_concurrent_downloads = max_concurrent_downloads
        self._sleep = sleep
        self._stats = DownloadStats()

    def get_target_path(self, post: Post) -> Path:
        stem, ext = os.path.splitext(os.path.basename(post.content_url))
        if not stem.startswith(str(post.id)):
            stem = str(post.id) + '_' + stem

        file_name = (
            '_'.join(
                str(part)
                for part in [self._plugin.name, stem, post.title]
                if part)
            + '.' + ext.lstrip('.'))
        return self._target_dir.joinpath(util.sanitize_file_name(file_name))

    async def download_file(self, post: Post) -> bool:
        target_path = self.get_target_path(post)

        stem, _ = os.path.splitext(os.path.basename(post.content_url))
        history_key = '{}/{}/{}'.format(self._plugin.name, post.id, stem)

        if target_path.exists():
            self._stats.ignore(post.content_url, 'already exists')
            return False

        if (
                self._history
                and not self._force
                and self._history.is_downloaded(history_key)):
            self._stats.ignore(post.content_url, 'already downloaded')
            return False

        content = None
        attempt = 0
        with self._stats.download(post.content_url):
            while True:
                attempt += 1
                try:
                    content = await self._plugin.get_post_content(post)
                    break
                except Exception:
                    if attempt > self._max_attempts:
                        raise

        target_path.parent.mkdir(parents=True, exist_ok=True)
        with target_path.open('wb') as handle:
            handle.write(content)

        if self._history:
            self._history.add(history_key)
            self._history.flush()

        return True

    async def run(self, query: str, limit: Optional[int]) -> None:
        downloaded = 0

        queue = asyncio.Queue(maxsize=self._max_concurrent_downloads)

        async def consumer():
            nonlocal downloaded
            while True:
                post = await queue.get()
                result = await self.download_file(post)
                if result:
                    await asyncio.sleep(self._sleep)
                    downloaded += 1
                queue.task_done()

        consumers = [
            asyncio.ensure_future(consumer())
            for i in range(self._max_concurrent_downloads)]

        try:
            async for post in self._plugin.find_posts(query):
                if limit is not None and downloaded >= limit:
                    break
                await queue.put(post)

            # wait untli complete
            await queue.join()
        finally:
            for consumer in consumers:
                consumer.cancel()

        print('Downloaded: {}'.format(self._stats.downloaded))
        print('Ignored: {}'.format(self._stats.ignored))
        print('Errors: {}'.format(self._stats.errors))

def parse_args() -> configargparse.Namespace:
    parser = cli.make_arg_parser(
        'Downloads posts from various boorus.', PLUGINS)
    parser.add(
        '-l', '--limit', default=None, type=int,
        help='limit how many files to download')
    parser.add(
        '-n', '--num', default=10, type=int,
        help='simultaneous jobs')
    parser.add(
        '--max-attempts', default=3, type=int,
        help='max attempts before abandoning download')
    parser.add(
        '-s', '--sleep', default=0, type=float,
        help='pause between uploads (in seconds)')
    parser.add(
        '--target-dir', default='~/{plugin}/{query}',
        help='where to put the files')
    parser.add(
        '--history-file', default='~/.cache/dl-booru.log',
        help='path to the history file')
    parser.add(
        '-f', '--force', action='store_true',
        help='redownload even if present in the history file')
    parser.add('query', help='query to filter the posts with')
    return parser.parse_args()

def main() -> None:
    args = parse_args()

    plugin: PluginBase = args.plugin
    user_name: str = args.user
    password: str = args.password

    force: bool = args.force
    max_attempts: int = args.max_attempts
    max_concurrent_downloads: int = args.num
    sleep: float = args.sleep
    limit: Optional[int] = args.limit
    query: str = args.query

    target_dir = Path(
        args.target_dir.format(
            plugin=plugin.name,
            query=util.sanitize_file_name(query))
        ).expanduser()

    history = (
        DownloadHistory(Path(args.history_file).expanduser())
        if args.history_file
        else None)

    try:
        loop = asyncio.get_event_loop()
        loop.run_until_complete(plugin.login(user_name, password))
        downloader = Downloader(
            plugin,
            target_dir,
            history,
            force,
            max_attempts,
            max_concurrent_downloads,
            sleep)
        loop.run_until_complete(downloader.run(query, limit))
    except KeyboardInterrupt:
        print('Aborted.')
    except errors.ApiError as ex:
        print('Error: %s' % str(ex), file=sys.stderr)

if __name__ == '__main__':
    main()
<commit_msg>opt/booru-toolkit: use bare file names<commit_after>#!/usr/bin/env python
import asyncio
import os
import sys
import contextlib
from pathlib import Path
from typing import Optional, List, Dict, Generator
import configargparse
from booru_toolkit import errors
from booru_toolkit import cli
from booru_toolkit import util
from booru_toolkit.plugin import PluginBase
from booru_toolkit.plugin import Post
from booru_toolkit.plugin import PluginGelbooru
from booru_toolkit.plugin import PluginPixiv
from booru_toolkit.plugin import PluginYume

PLUGINS: List[PluginBase] = [PluginGelbooru(), PluginPixiv(), PluginYume()]

class DownloadHistory:
    def __init__(self, path: Path) -> None:
        self._path = path
        self._downloaded_urls: Dict[str, bool] = {}
        self._buffered_urls: List[str] = []
        if self._path.exists():
            for line in self._path.open('r'):
                self._downloaded_urls[line.strip()] = True

    def add(self, url: str) -> None:
        if self.is_downloaded(url):
            return
        self._downloaded_urls[url] = True
        self._buffered_urls.append(url)

    def flush(self) -> None:
        self._path.parent.mkdir(parents=True, exist_ok=True)
        with self._path.open('a') as handle:
            for url in self._buffered_urls:
                print(url, file=handle)
        self._buffered_urls = []

    def is_downloaded(self, url: str) -> bool:
        return url in self._downloaded_urls

class DownloadStats:
    def __init__(self) -> None:
        self.errors = 0
        self.ignored = 0
        self.downloaded = 0

    def ignore(self, url: str, reason: str) -> None:
        print('{}: ignored ({})'.format(url, reason))
        self.ignored += 1

    @contextlib.contextmanager
    def download(self, url: str) -> Generator:
        try:
            print('{}: downloading...'.format(url))
            yield
            self.downloaded += 1
            print('{}: saved'.format(url))
        except Exception as ex:
            print('error: {}'.format(ex))
            self.errors += 1
            raise

class Downloader:
    def __init__(
            self,
            plugin: PluginBase,
            target_dir: Path,
            history: Optional[DownloadHistory],
            force: bool,
            max_attempts: int,
            max_concurrent_downloads: int,
            sleep: float) -> None:
        self._plugin = plugin
        self._target_dir = target_dir
        self._history = history
        self._force = force
        self._max_attempts = max_attempts
        self._max_concurrent_downloads = max_concurrent_downloads
        self._sleep = sleep
        self._stats = DownloadStats()

    def get_target_path(self, post: Post) -> Path:
        stem, ext = os.path.splitext(os.path.basename(post.content_url))
        if not stem.startswith(str(post.id)):
            stem = str(post.id) + '_' + stem

        file_name = (
            '_'.join(
                str(part)
                for part in [self._plugin.name, stem]
                if part)
            + '.' + ext.lstrip('.'))
        return self._target_dir.joinpath(util.sanitize_file_name(file_name))

    async def download_file(self, post: Post) -> bool:
        target_path = self.get_target_path(post)

        stem, _ = os.path.splitext(os.path.basename(post.content_url))
        history_key = '{}/{}/{}'.format(self._plugin.name, post.id, stem)

        if target_path.exists():
            self._stats.ignore(post.content_url, 'already exists')
            return False

        if (
                self._history
                and not self._force
                and self._history.is_downloaded(history_key)):
            self._stats.ignore(post.content_url, 'already downloaded')
            return False

        content = None
        attempt = 0
        with self._stats.download(post.content_url):
            while True:
                attempt += 1
                try:
                    content = await self._plugin.get_post_content(post)
                    break
                except Exception:
                    if attempt > self._max_attempts:
                        raise

        target_path.parent.mkdir(parents=True, exist_ok=True)
        with target_path.open('wb') as handle:
            handle.write(content)

        if self._history:
            self._history.add(history_key)
            self._history.flush()

        return True

    async def run(self, query: str, limit: Optional[int]) -> None:
        downloaded = 0

        queue = asyncio.Queue(maxsize=self._max_concurrent_downloads)

        async def consumer():
            nonlocal downloaded
            while True:
                post = await queue.get()
                result = await self.download_file(post)
                if result:
                    await asyncio.sleep(self._sleep)
                    downloaded += 1
                queue.task_done()

        consumers = [
            asyncio.ensure_future(consumer())
            for i in range(self._max_concurrent_downloads)]

        try:
            async for post in self._plugin.find_posts(query):
                if limit is not None and downloaded >= limit:
                    break
                await queue.put(post)

            # wait untli complete
            await queue.join()
        finally:
            for consumer in consumers:
                consumer.cancel()

        print('Downloaded: {}'.format(self._stats.downloaded))
        print('Ignored: {}'.format(self._stats.ignored))
        print('Errors: {}'.format(self._stats.errors))

def parse_args() -> configargparse.Namespace:
    parser = cli.make_arg_parser(
        'Downloads posts from various boorus.', PLUGINS)
    parser.add(
        '-l', '--limit', default=None, type=int,
        help='limit how many files to download')
    parser.add(
        '-n', '--num', default=10, type=int,
        help='simultaneous jobs')
    parser.add(
        '--max-attempts', default=3, type=int,
        help='max attempts before abandoning download')
    parser.add(
        '-s', '--sleep', default=0, type=float,
        help='pause between uploads (in seconds)')
    parser.add(
        '--target-dir', default='~/{plugin}/{query}',
        help='where to put the files')
    parser.add(
        '--history-file', default='~/.cache/dl-booru.log',
        help='path to the history file')
    parser.add(
        '-f', '--force', action='store_true',
        help='redownload even if present in the history file')
    parser.add('query', help='query to filter the posts with')
    return parser.parse_args()

def main() -> None:
    args = parse_args()

    plugin: PluginBase = args.plugin
    user_name: str = args.user
    password: str = args.password

    force: bool = args.force
    max_attempts: int = args.max_attempts
    max_concurrent_downloads: int = args.num
    sleep: float = args.sleep
    limit: Optional[int] = args.limit
    query: str = args.query

    target_dir = Path(
        args.target_dir.format(
            plugin=plugin.name,
            query=util.sanitize_file_name(query))
        ).expanduser()

    history = (
        DownloadHistory(Path(args.history_file).expanduser())
        if args.history_file
        else None)

    try:
        loop = asyncio.get_event_loop()
        loop.run_until_complete(plugin.login(user_name, password))
        downloader = Downloader(
            plugin,
            target_dir,
            history,
            force,
            max_attempts,
            max_concurrent_downloads,
            sleep)
        loop.run_until_complete(downloader.run(query, limit))
    except KeyboardInterrupt:
        print('Aborted.')
    except errors.ApiError as ex:
        print('Error: %s' % str(ex), file=sys.stderr)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import csv, subprocess

class StringBuilder(object):
  def __init__(self, item=None):
    self.items = []
    if item is not None:
      self.add(item)

  def add(self, item):
    self.items.append(str(item))
    return self

  def __str__(self):
    return "".join(self.items)

target_map = """android-slices-builders,androidx.slice_slice-builders
android-slices-core,androidx.slice_slice-core
android-slices-view,androidx.slice_slice-view
android-support-animatedvectordrawable,androidx.vectordrawable_vectordrawable-animated
android-support-annotations,androidx.annotation_annotation
android-support-asynclayoutinflater,androidx.asynclayoutinflater_asynclayoutinflater
android-support-car,androidx.car_car
android-support-collections,androidx.collection_collection
android-support-compat,androidx.core_core
android-support-constraint-layout,androidx-constraintlayout_constraintlayout
android-support-constraint-layout-solver,androidx-constraintlayout_constraintlayout-solver
android-support-contentpaging,androidx.contentpager_contentpager
android-support-coordinatorlayout,androidx.coordinatorlayout_coordinatorlayout
android-support-core-ui,androidx.legacy_legacy-support-core-ui
android-support-core-utils,androidx.legacy_legacy-support-core-utils
android-support-cursoradapter,androidx.cursoradapter_cursoradapter
android-support-customtabs,androidx.browser_browser
android-support-customview,androidx.customview_customview
android-support-documentfile,androidx.documentfile_documentfile
android-support-drawerlayout,androidx.drawerlayout_drawerlayout
android-support-dynamic-animation,androidx.dynamicanimation_dynamicanimation
android-support-emoji,androidx.emoji_emoji
android-support-emoji-appcompat,androidx.emoji_emoji-appcompat
android-support-emoji-bundled,androidx.emoji_emoji-bundled
android-support-exifinterface,androidx.exifinterface_exifinterface
android-support-fragment,androidx.fragment_fragment
android-support-heifwriter,androidx.heifwriter_heifwriter
android-support-interpolator,androidx.interpolator_interpolator
android-support-loader,androidx.loader_loader
android-support-localbroadcastmanager,androidx.localbroadcastmanager_localbroadcastmanager
android-support-media-compat,androidx.media_media
android-support-percent,androidx.percentlayout_percentlayout
android-support-print,androidx.print_print
android-support-recommendation,androidx.recommendation_recommendation
android-support-recyclerview-selection,androidx.recyclerview_recyclerview-selection
android-support-slidingpanelayout,androidx.slidingpanelayout_slidingpanelayout
android-support-swiperefreshlayout,androidx.swiperefreshlayout_swiperefreshlayout
android-support-textclassifier,androidx.textclassifier_textclassifier
android-support-transition,androidx.transition_transition
android-support-tv-provider,androidx.tvprovider_tvprovider
android-support-v13,androidx.legacy_legacy-support-v13
android-support-v14-preference,androidx.legacy_legacy-preference-v14
android-support-v17-leanback,androidx.leanback_leanback
android-support-v17-preference-leanback,androidx.leanback_leanback-preference
android-support-v4,androidx.legacy_legacy-support-v4
android-support-v7-appcompat,androidx.appcompat_appcompat
android-support-v7-cardview,androidx.cardview_cardview
android-support-v7-gridlayout,androidx.gridlayout_gridlayout
android-support-v7-mediarouter,androidx.mediarouter_mediarouter
android-support-v7-palette,androidx.palette_palette
android-support-v7-preference,androidx.preference_preference
android-support-v7-recyclerview,androidx.recyclerview_recyclerview
android-support-vectordrawable,androidx.vectordrawable_vectordrawable
android-support-viewpager,androidx.viewpager_viewpager
android-support-wear,androidx.wear_wear
android-support-webkit,androidx.webkit_webkit
android-arch-core-common,androidx.arch.core_core-common
android-arch-core-runtime,androidx.arch.core_core-runtime
android-arch-lifecycle-common,androidx.lifecycle_lifecycle-common
android-arch-lifecycle-common-java8,androidx.lifecycle_lifecycle-common-java8
android-arch-lifecycle-extensions,androidx.lifecycle_lifecycle-extensions
android-arch-lifecycle-livedata,androidx.lifecycle_lifecycle-livedata
android-arch-lifecycle-livedata-core,androidx.lifecycle_lifecycle-livedata-core
android-arch-lifecycle-runtime,androidx.lifecycle_lifecycle-runtime
android-arch-lifecycle-viewmodel,androidx.lifecycle_lifecycle-viewmodel
android-arch-paging-common,androidx.paging_paging-common
android-arch-paging-runtime,androidx.paging_paging-runtime
android-arch-persistence-db,androidx.sqlite_sqlite
android-arch-persistence-db-framework,androidx.sqlite_sqlite-framework
android-arch-room-common,androidx.room_room-common
android-arch-room-migration,androidx.room_room-migration
android-arch-room-runtime,androidx.room_room-runtime
android-arch-room-testing,androidx.room_room-testing
$(ANDROID_SUPPORT_DESIGN_TARGETS),androidx.design_design"""

reader = csv.reader(target_map.split('\n'), delimiter=',')

rewriterTextBuilder = StringBuilder()
for row in reader:
  rewriterTextBuilder.add("s|").add(row[0]).add("|").add(row[1]).add("|g\n")
scriptPath = "/tmp/jetifier-make-sed-script.txt"
print("Writing " + scriptPath)
with open(scriptPath, 'w') as scriptFile:
  scriptFile.write(str(rewriterTextBuilder))

rewriteCommand = "time find . -name out -prune -o -name .git -prune -o -name .repo -prune -o -iregex '.*\.mk' -print | xargs -n 1 --no-run-if-empty -P 64 sed -i -f /tmp/jetifier-make-sed-script.txt"

print("""
Will run command:

""" + rewriteCommand + """

""")
response = raw_input("Ok? [y/n]")
if response == "y":
  subprocess.check_output(rewriteCommand, shell=True)

<commit_msg>Update migration script to use correct material target<commit_after>#!/usr/bin/python

import csv, subprocess

class StringBuilder(object):
  def __init__(self, item=None):
    self.items = []
    if item is not None:
      self.add(item)

  def add(self, item):
    self.items.append(str(item))
    return self

  def __str__(self):
    return "".join(self.items)

target_map = """android-slices-builders,androidx.slice_slice-builders
android-slices-core,androidx.slice_slice-core
android-slices-view,androidx.slice_slice-view
android-support-animatedvectordrawable,androidx.vectordrawable_vectordrawable-animated
android-support-annotations,androidx.annotation_annotation
android-support-asynclayoutinflater,androidx.asynclayoutinflater_asynclayoutinflater
android-support-car,androidx.car_car
android-support-collections,androidx.collection_collection
android-support-compat,androidx.core_core
android-support-constraint-layout,androidx-constraintlayout_constraintlayout
android-support-constraint-layout-solver,androidx-constraintlayout_constraintlayout-solver
android-support-contentpaging,androidx.contentpager_contentpager
android-support-coordinatorlayout,androidx.coordinatorlayout_coordinatorlayout
android-support-core-ui,androidx.legacy_legacy-support-core-ui
android-support-core-utils,androidx.legacy_legacy-support-core-utils
android-support-cursoradapter,androidx.cursoradapter_cursoradapter
android-support-customtabs,androidx.browser_browser
android-support-customview,androidx.customview_customview
android-support-documentfile,androidx.documentfile_documentfile
android-support-drawerlayout,androidx.drawerlayout_drawerlayout
android-support-dynamic-animation,androidx.dynamicanimation_dynamicanimation
android-support-emoji,androidx.emoji_emoji
android-support-emoji-appcompat,androidx.emoji_emoji-appcompat
android-support-emoji-bundled,androidx.emoji_emoji-bundled
android-support-exifinterface,androidx.exifinterface_exifinterface
android-support-fragment,androidx.fragment_fragment
android-support-heifwriter,androidx.heifwriter_heifwriter
android-support-interpolator,androidx.interpolator_interpolator
android-support-loader,androidx.loader_loader
android-support-localbroadcastmanager,androidx.localbroadcastmanager_localbroadcastmanager
android-support-media-compat,androidx.media_media
android-support-percent,androidx.percentlayout_percentlayout
android-support-print,androidx.print_print
android-support-recommendation,androidx.recommendation_recommendation
android-support-recyclerview-selection,androidx.recyclerview_recyclerview-selection
android-support-slidingpanelayout,androidx.slidingpanelayout_slidingpanelayout
android-support-swiperefreshlayout,androidx.swiperefreshlayout_swiperefreshlayout
android-support-textclassifier,androidx.textclassifier_textclassifier
android-support-transition,androidx.transition_transition
android-support-tv-provider,androidx.tvprovider_tvprovider
android-support-v13,androidx.legacy_legacy-support-v13
android-support-v14-preference,androidx.legacy_legacy-preference-v14
android-support-v17-leanback,androidx.leanback_leanback
android-support-v17-preference-leanback,androidx.leanback_leanback-preference
android-support-v4,androidx.legacy_legacy-support-v4
android-support-v7-appcompat,androidx.appcompat_appcompat
android-support-v7-cardview,androidx.cardview_cardview
android-support-v7-gridlayout,androidx.gridlayout_gridlayout
android-support-v7-mediarouter,androidx.mediarouter_mediarouter
android-support-v7-palette,androidx.palette_palette
android-support-v7-preference,androidx.preference_preference
android-support-v7-recyclerview,androidx.recyclerview_recyclerview
android-support-vectordrawable,androidx.vectordrawable_vectordrawable
android-support-viewpager,androidx.viewpager_viewpager
android-support-wear,androidx.wear_wear
android-support-webkit,androidx.webkit_webkit
android-arch-core-common,androidx.arch.core_core-common
android-arch-core-runtime,androidx.arch.core_core-runtime
android-arch-lifecycle-common,androidx.lifecycle_lifecycle-common
android-arch-lifecycle-common-java8,androidx.lifecycle_lifecycle-common-java8
android-arch-lifecycle-extensions,androidx.lifecycle_lifecycle-extensions
android-arch-lifecycle-livedata,androidx.lifecycle_lifecycle-livedata
android-arch-lifecycle-livedata-core,androidx.lifecycle_lifecycle-livedata-core
android-arch-lifecycle-runtime,androidx.lifecycle_lifecycle-runtime
android-arch-lifecycle-viewmodel,androidx.lifecycle_lifecycle-viewmodel
android-arch-paging-common,androidx.paging_paging-common
android-arch-paging-runtime,androidx.paging_paging-runtime
android-arch-persistence-db,androidx.sqlite_sqlite
android-arch-persistence-db-framework,androidx.sqlite_sqlite-framework
android-arch-room-common,androidx.room_room-common
android-arch-room-migration,androidx.room_room-migration
android-arch-room-runtime,androidx.room_room-runtime
android-arch-room-testing,androidx.room_room-testing
$(ANDROID_SUPPORT_DESIGN_TARGETS),com.google.android.material_material"""

reader = csv.reader(target_map.split('\n'), delimiter=',')

rewriterTextBuilder = StringBuilder()
for row in reader:
  rewriterTextBuilder.add("s|").add(row[0]).add("|").add(row[1]).add("|g\n")
scriptPath = "/tmp/jetifier-make-sed-script.txt"
print("Writing " + scriptPath)
with open(scriptPath, 'w') as scriptFile:
  scriptFile.write(str(rewriterTextBuilder))

rewriteCommand = "time find . -name out -prune -o -name .git -prune -o -name .repo -prune -o -iregex '.*\.mk' -print | xargs -n 1 --no-run-if-empty -P 64 sed -i -f /tmp/jetifier-make-sed-script.txt"

print("""
Will run command:

""" + rewriteCommand + """

""")
response = raw_input("Ok? [y/n]")
if response == "y":
  subprocess.check_output(rewriteCommand, shell=True)

<|endoftext|>


<commit_before>from pygments.lexer import RegexLexer, bygroups, include
from pygments.formatters import HtmlFormatter
from pygments.formatters.html import _escape_html_table
from pygments.style import Style
from pygments.token import *
import re

DATE_RE = r'\d\d? [A-Z][a-z]{2} \d{4}'
EMAIL_RE = r'[\w\.\-\+]+@(?:[\w\-]+\.)+\w+'
LINK_RE = r'https?:\/\/(?:[\w\-]+\.)+\w+(:?\/[\w\/\.\-\_\+\&\%]+)?'

ARCHES = [ u'alpha', u'amd64', u'amd64-fbsd', u'amd64-linux', u'arm',
           u'arm-linux', u'hppa', u'hppa-hpux', u'ia64', u'ia64-hpux',
           u'ia64-linux', u'm68k', u'm68k-mint', u'mips', u'ppc', u'ppc-aix',
           u'ppc-macos', u'ppc-openbsd', u'ppc64', u's390', u'sh', u'sparc',
           u'sparc-fbsd', u'sparc-solaris', u'sparc64-solaris', u'x64-freebsd',
           u'x64-macos', u'x64-openbsd', u'x64-solaris', u'x86', u'x86-fbsd',
           u'x86-freebsd', u'x86-interix', u'x86-linux', u'x86-macos',
           u'x86-netbsd', u'x86-openbsd', u'x86-solaris', u'x86-winnt' ]

KEYWORD_RE = r'[-~]?(?:%s)' % '|'.join(map(re.escape, ARCHES))

# Literal.Date

Date = Literal.Date
PackageName = Name.Namespace
FilePlus = Generic.Inserted
FileMinus = Generic.Deleted
File = Operator.Word
Email = Number
AuthorName = Name.Variable
Bug = String
Link = Keyword

class ChangelogLexer(RegexLexer):
    name = 'Changelog'
    aliases = ['changelog']
    filenames = ['Changelog']

    tokens = {
        'root': [
            (r'^# .*\n', Comment.Single), # Comment
            (r'^(\*)(.+)( )(\()(%s)(\))' % DATE_RE, bygroups(Operator, PackageName, Whitespace, Punctuation, Date, Punctuation)),
            (r'^(  )(%(date)s)(;)( +)([^<]*)(<)(%(email)s)(>)' % {'date': DATE_RE, 'email': EMAIL_RE},
                bygroups(Whitespace, Date, Punctuation, Whitespace, AuthorName, Punctuation, Number, Punctuation), 'main'), # Date
            (EMAIL_RE, Email),
            (r' ', Whitespace),
            include('bugs'),
            (r'\(|\)|<|>', Punctuation),
            include('email'),
            include('link'),
            #(KEYWORD_RE, Name.Variable),
            (r'[^#\n ]+', Generic),
            (r' +', Whitespace),
        ],
        'main': [
            (r' *\n\n', Punctuation, '#pop'),
            (EMAIL_RE, Email),
            (r'(,| |  )(\+[\w\.\-\+\/]+)', bygroups(Punctuation, FilePlus)),
            (r'(,| |  )(\-[\w\.\-\+\/]+)', bygroups(Punctuation, FileMinus)),
            (r'(,| |  )([\w\*][\w\.\-\+\/]+)', bygroups(Punctuation, File)),
            (r': *\n *\n|:(?: +|\n)|: +', Punctuation, 'message'),
            (r';|<|>|,', Punctuation),
            include('link'),
            (r' |\n', Whitespace),
        ],
        'message': [
            (r' *\n', Punctuation, '#pop:2'),
            #(r'#\d+', String),
            (r'  ', Whitespace, 'message_text'),
        ],
        'message_text': [
            (r' *\n', Whitespace, '#pop'),
            include('bugs'),
            (r'\(|\)|<|>', Punctuation),
            include('email'),
            include('link'),
            #(KEYWORD_RE, Name.Variable),
            (r' +', Whitespace),
            (r'.', Generic),
        ],
        'bugs' : [
            (r'(?i)bug #\d+', Bug),
            (r'#\d+', Bug),
            (r'(?i)bug \d+', Bug),
        ],
        'email': [
            (EMAIL_RE, Email),
        ],
        'link': [
            (LINK_RE, Link),
        ]
    }

class ChangelogHtmlFormater(HtmlFormatter):
    def _format_lines(self, tokensource):
        """
        Copyed from pygments source litle modified.
        """
        nocls = self.noclasses
        lsep = self.lineseparator
        # for  lookup only
        getcls = self.ttype2class.get
        c2s = self.class2style
        escape_table = _escape_html_table

        lspan = ''
        line = ''
        for ttype, value in tokensource:
            if nocls:
                cclass = getcls(ttype)
                while cclass is None:
                    ttype = ttype.parent
                    cclass = getcls(ttype)
                cspan = cclass and '' % c2s[cclass][0] or ''
            else:
                cls = self._get_css_class(ttype)
                cspan = cls and '' % cls or ''

            parts = value.translate(escape_table).split('\n')

            # for all but the last line
            for part in parts[:-1]:
                part = self.token_decorate(ttype, part)
                if line:
                    if lspan != cspan:
                        line += (lspan and '') + cspan + part + \
                                (cspan and '') + lsep
                    else: # both are the same
                        line += part + (lspan and '') + lsep
                    yield 1, line
                    line = ''
                elif part:
                    yield 1, cspan + part + (cspan and '') + lsep
                else:
                    yield 1, lsep
            # for the last line
            last = self.token_decorate(ttype, parts[-1])
            if line and last:
                if lspan != cspan:
                    line += (lspan and '') + cspan + last
                    lspan = cspan
                else:
                    line += last
            elif last:
                line = cspan + last
                lspan = cspan
            # else we neither have to open a new span nor set lspan

        if line:
            yield 1, line + (lspan and '') + lsep

    def token_decorate(self, token, value):
        if token == Link:
            value = '{0}'.format(value)
        return value

class ChangelogStyle(Style):
    default_style = ""
    styles = {
        Comment:                'italic #888',
        Date:                   '#F90',
        FilePlus:               'bold #0F0',
        FileMinus:              'bold #F00',
        File:                   'bold #C0A',
        PackageName:            'bold',
        Bug:                    '#E00',
        Email:                  '#0CF',
        AuthorName:             'bold #1C9',
    }
<commit_msg>Add highlight function<commit_after>from pygments.lexer import RegexLexer, bygroups, include
from pygments.formatters import HtmlFormatter, Terminal256Formatter
from pygments.formatters.html import _escape_html_table
from pygments.style import Style
from pygments.token import *
from pygments import highlight
import re

DATE_RE = r'\d\d? [A-Z][a-z]{2} \d{4}'
EMAIL_RE = r'[\w\.\-\+]+@(?:[\w\-]+\.)+\w+'
LINK_RE = r'https?:\/\/(?:[\w\-]+\.)+\w+(:?\/[\w\/\.\-\_\+\&\%]+)?'

ARCHES = [ u'alpha', u'amd64', u'amd64-fbsd', u'amd64-linux', u'arm',
           u'arm-linux', u'hppa', u'hppa-hpux', u'ia64', u'ia64-hpux',
           u'ia64-linux', u'm68k', u'm68k-mint', u'mips', u'ppc', u'ppc-aix',
           u'ppc-macos', u'ppc-openbsd', u'ppc64', u's390', u'sh', u'sparc',
           u'sparc-fbsd', u'sparc-solaris', u'sparc64-solaris', u'x64-freebsd',
           u'x64-macos', u'x64-openbsd', u'x64-solaris', u'x86', u'x86-fbsd',
           u'x86-freebsd', u'x86-interix', u'x86-linux', u'x86-macos',
           u'x86-netbsd', u'x86-openbsd', u'x86-solaris', u'x86-winnt' ]

KEYWORD_RE = r'[-~]?(?:%s)' % '|'.join(map(re.escape, ARCHES))

# Literal.Date

Date = Literal.Date
PackageName = Name.Namespace
FilePlus = Generic.Inserted
FileMinus = Generic.Deleted
File = Operator.Word
Email = Number
AuthorName = Name.Variable
Bug = String
Link = Keyword

class ChangelogLexer(RegexLexer):
    name = 'Changelog'
    aliases = ['changelog']
    filenames = ['Changelog']

    tokens = {
        'root': [
            (r'^# .*\n', Comment.Single), # Comment
            (r'^(\*)(.+)( )(\()(%s)(\))' % DATE_RE, bygroups(Operator, PackageName, Whitespace, Punctuation, Date, Punctuation)),
            (r'^(  )(%(date)s)(;)( +)([^<]*)(<)(%(email)s)(>)' % {'date': DATE_RE, 'email': EMAIL_RE},
                bygroups(Whitespace, Date, Punctuation, Whitespace, AuthorName, Punctuation, Number, Punctuation), 'main'), # Date
            include('email'),
            (r' ', Whitespace),
            include('bugs'),
            (r'\(|\)|<|>', Punctuation),
            include('email'),
            include('link'),
            #(KEYWORD_RE, Name.Variable),
            (r'[^#\n ]+', Generic),
            (r' +', Whitespace),
        ],
        'main': [
            (r' *\n\n', Punctuation, '#pop'),
            (EMAIL_RE, Email),
            (r'(,| |  )(\+[\w\.\-\+\/]+)', bygroups(Punctuation, FilePlus)),
            (r'(,| |  )(\-[\w\.\-\+\/]+)', bygroups(Punctuation, FileMinus)),
            (r'(,| |  )([\w\*][\w\.\-\+\/]+)', bygroups(Punctuation, File)),
            (r': *\n *\n|:(?: +|\n)|: +', Punctuation, 'message'),
            (r';|<|>|,', Punctuation),
            include('link'),
            (r' |\n', Whitespace),
        ],
        'message': [
            (r' *\n', Punctuation, '#pop:2'),
            #(r'#\d+', String),
            (r'  ', Whitespace, 'message_text'),
        ],
        'message_text': [
            (r' *\n', Whitespace, '#pop'),
            include('bugs'),
            include('email'),
            include('link'),
            #(KEYWORD_RE, Name.Variable),
            (r' +', Whitespace),
            (r'.', Generic),
        ],
        'bugs' : [
            (r'(?i)bug #\d+', Bug),
            (r'#\d+', Bug),
            (r'(?i)bug \d+', Bug),
        ],
        'email': [
            (EMAIL_RE, Email),
        ],
        'link': [
            (LINK_RE, Link),
        ]
    }

class ChangelogHtmlFormater(HtmlFormatter):
    def _format_lines(self, tokensource):
        """
        Copyed from pygments source litle modified.
        """
        nocls = self.noclasses
        lsep = self.lineseparator
        # for  lookup only
        getcls = self.ttype2class.get
        c2s = self.class2style
        escape_table = _escape_html_table

        lspan = ''
        line = ''
        for ttype, value in tokensource:
            if nocls:
                cclass = getcls(ttype)
                while cclass is None:
                    ttype = ttype.parent
                    cclass = getcls(ttype)
                cspan = cclass and '' % c2s[cclass][0] or ''
            else:
                cls = self._get_css_class(ttype)
                cspan = cls and '' % cls or ''

            parts = value.translate(escape_table).split('\n')

            # for all but the last line
            for part in parts[:-1]:
                part = self.token_decorate(ttype, part)
                if line:
                    if lspan != cspan:
                        line += (lspan and '') + cspan + part + \
                                (cspan and '') + lsep
                    else: # both are the same
                        line += part + (lspan and '') + lsep
                    yield 1, line
                    line = ''
                elif part:
                    yield 1, cspan + part + (cspan and '') + lsep
                else:
                    yield 1, lsep
            # for the last line
            last = self.token_decorate(ttype, parts[-1])
            if line and last:
                if lspan != cspan:
                    line += (lspan and '') + cspan + last
                    lspan = cspan
                else:
                    line += last
            elif last:
                line = cspan + last
                lspan = cspan
            # else we neither have to open a new span nor set lspan

        if line:
            yield 1, line + (lspan and '') + lsep

    def token_decorate(self, token, value):
        if token == Link:
            value = '{0}'.format(value)
        return value

class ChangelogStyle(Style):
    default_style = ""
    styles = {
        Comment:                'italic #888',
        Date:                   '#F90',
        FilePlus:               'bold #0F0',
        FileMinus:              'bold #F00',
        File:                   'bold #C0A',
        PackageName:            'bold',
        Bug:                    '#E00',
        Email:                  '#0CF',
        AuthorName:             'bold #1C9',
        #Error:                  'bold underline #F00',
    }

def changelog_highlight(text):
    return highlight(text, ChangelogLexer(),
                     ChangelogHtmlFormater(style = ChangelogStyle))

def changelog_termial_highlight(text):
    return highlight(text, ChangelogLexer(),
                     Terminal256Formatter(style = ChangelogStyle))
<|endoftext|>


<commit_before># coding=utf-8

# Progetto: Selfie-O-Matic

import sys, os
import time
import io

try:
    from picamera.array import PiRGBArray
    from picamera import PiCamera
except:
    pass

import cv2
from cv2 import VideoCapture
import numpy as np

import thread

from fb import post_image

import logging
import settings

from image_lib import overlay_image, fadein, create_empty_image

from tasks.task_countdown import CountdownTask
from tasks.task_fadetowhite import FadeToWhiteTask
from tasks.task_snapshot import SnapShotTask
from tasks.task_postonfb import PostOnFbTask

__author__ = "Fabrizio Guglielmino"

class DeviceContext(object):
    camera = None
    cap = None

    def __init__(self, camera, cap):
        self.camera = camera
        self.cap = cap

class SelfieOMatic(object):
    _is_running = False
    # TODO: Coda di frame processors
    _processors = []
    rawCapture = None
    ctx = DeviceContext(None, None)

    def __init__(self, device=0):

        self.ctx.camera = None
        self.cap = None
        try:
            cv2.namedWindow("MainWin")

            self.ctx.camera = PiCamera()
            self.ctx.camera.start_preview()
            self.ctx.camera.framerate = 24
            #self.ctx.camera.resolution = (640, 480)
            self.ctx.camera.preview_fullscreen = True
            #self.ctx.camera.preview_window = (0, 0, 640, 480)
            self.rawCapture = PiRGBArray(self.ctx.camera)
            time.sleep(0.3)

        except:
            print "fallback OpenCV standard"
            self.cap = cv2.VideoCapture(device)

    def run(self):
        self._is_running = True
        while self._is_running:
            frame = self.__get_frame()
            self.__process_input()
            frame = self.__process_frame(frame)
            self.__show_frame(frame)
            time.sleep(0.05)

    def cleanup(self):
        if self.cap :
            self.cap.release()
        if self.ctx.camera:
            self.ctx.camera.stop_preview()
            self.ctx.camera.close()
        cv2.destroyAllWindows()

    def __get_frame(self):
        frame = None
        if self.ctx.camera:
            #img = self.ctx.camera.capture_continuous(self.rawCapture, format="bgr", use_video_port=False)
            #frame = np.array(img.next().array, copy=True)
            pass
        else:
            ret, frame = self.cap.read()
        return frame

    def __process_input(self):
        key = cv2.waitKey(10)
        if key == ord('q'):
            cv2.destroyAllWindows()
            self._is_running = False

        elif key == ord('s'):
            processor = CountdownTask(self.ctx)
            self._processors.append(processor)

            fade = FadeToWhiteTask(self.ctx)
            self._processors.append(fade)

            snap = SnapShotTask(self.ctx)
            self._processors.append(snap)

            #postfb = PostOnFbTask(self.ctx)
            #self._processors.append(postfb)

    def __process_frame(self, frame):
        if len(self._processors) > 0:
            processor = self._processors[0]
            if processor.is_completed():
                self._processors.remove(processor)
            else:
                frame = processor.process_frame(frame)
        return frame

    def __show_frame(self, frame):
        # Camera uses "start_preview"
        if self.ctx.camera is None:
            cv2.imshow(settings.APP_NAME, frame)
        else:
            self.rawCapture.truncate(0)

if __name__ == '__main__':

    selfie = SelfieOMatic()
    selfie.run()
    selfie.cleanup()
<commit_msg>Updated snapshot task<commit_after># coding=utf-8

# Progetto: Selfie-O-Matic

import sys, os
import time
import io

try:
    from picamera.array import PiRGBArray
    from picamera import PiCamera
except:
    pass

import cv2
from cv2 import VideoCapture
import numpy as np

import thread

from fb import post_image

import logging
import settings

from image_lib import overlay_image, fadein, create_empty_image

from tasks.task_countdown import CountdownTask
from tasks.task_fadetowhite import FadeToWhiteTask
from tasks.task_snapshot import SnapShotTask
from tasks.task_postonfb import PostOnFbTask

__author__ = "Fabrizio Guglielmino"

class DeviceContext(object):
    camera = None
    cap = None

    def __init__(self, camera, cap):
        self.camera = camera
        self.cap = cap

class SelfieOMatic(object):
    _is_running = False
    # TODO: Coda di frame processors
    _processors = []
    rawCapture = None
    ctx = DeviceContext(None, None)

    def __init__(self, device=0):

        self.ctx.camera = None
        self.cap = None
        try:
            cv2.namedWindow("MainWin")

            self.ctx.camera = PiCamera()

            self.ctx.camera.framerate = 24
            #self.ctx.camera.resolution = (640, 480)
            self.ctx.camera.preview_fullscreen = True
            #self.ctx.camera.preview_window = (0, 0, 640, 480)
            self.ctx.camera.start_preview()
            self.rawCapture = PiRGBArray(self.ctx.camera)
            time.sleep(0.3)

        except:
            print "fallback OpenCV standard"
            self.cap = cv2.VideoCapture(device)

    def run(self):
        self._is_running = True
        while self._is_running:
            frame = self.__get_frame()
            self.__process_input()
            frame = self.__process_frame(frame)
            self.__show_frame(frame)
            time.sleep(0.05)

    def cleanup(self):
        if self.cap :
            self.cap.release()
        if self.ctx.camera:
            self.ctx.camera.stop_preview()
            self.ctx.camera.close()
        cv2.destroyAllWindows()

    def __get_frame(self):
        frame = None
        if self.ctx.camera:
            #img = self.ctx.camera.capture_continuous(self.rawCapture, format="bgr", use_video_port=False)
            #frame = np.array(img.next().array, copy=True)
            pass
        else:
            ret, frame = self.cap.read()
        return frame

    def __process_input(self):
        key = cv2.waitKey(10)
        if key == ord('q'):
            cv2.destroyAllWindows()
            self._is_running = False

        elif key == ord('s'):
            processor = CountdownTask(self.ctx)
            self._processors.append(processor)

            fade = FadeToWhiteTask(self.ctx)
            self._processors.append(fade)

            snap = SnapShotTask(self.ctx)
            self._processors.append(snap)

            #postfb = PostOnFbTask(self.ctx)
            #self._processors.append(postfb)

    def __process_frame(self, frame):
        if len(self._processors) > 0:
            processor = self._processors[0]
            if processor.is_completed():
                self._processors.remove(processor)
            else:
                frame = processor.process_frame(frame)
        return frame

    def __show_frame(self, frame):
        # Camera uses "start_preview"
        if self.ctx.camera is None:
            cv2.imshow(settings.APP_NAME, frame)
        else:
            self.rawCapture.truncate(0)

if __name__ == '__main__':

    selfie = SelfieOMatic()
    selfie.run()
    selfie.cleanup()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
#                                                       DIT, UPM
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
"""
Senpy is a modular sentiment analysis server. This script runs an instance of
the server.

"""

from flask import Flask
from senpy.extensions import Senpy
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

import logging
import os
import argparse
import senpy

SERVER_PORT = os.environ.get("PORT", 5000)

def main():
    parser = argparse.ArgumentParser(description='Run a Senpy server')
    parser.add_argument(
        '--level',
        '-l',
        metavar='logging_level',
        type=str,
        default="INFO",
        help='Logging level')
    parser.add_argument(
        '--debug',
        '-d',
        action='store_true',
        default=False,
        help='Run the application in debug mode')
    parser.add_argument(
        '--default-plugins',
        action='store_true',
        default=False,
        help='Load the default plugins')
    parser.add_argument(
        '--host',
        type=str,
        default="0.0.0.0",
        help='Use 0.0.0.0 to accept requests from any host.')
    parser.add_argument(
        '--port',
        '-p',
        type=int,
        default=SERVER_PORT,
        help='Port to listen on.')
    parser.add_argument(
        '--plugins-folder',
        '-f',
        type=str,
        default='plugins',
        help='Where to look for plugins.')
    parser.add_argument(
        '--only-install',
        '-i',
        action='store_true',
        default=False,
        help='Do not run a server, only install plugin dependencies')
    args = parser.parse_args()
    logging.basicConfig()
    rl = logging.getLogger()
    rl.setLevel(getattr(logging, args.level))
    app = Flask(__name__)
    app.debug = args.debug
    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
    if args.only_install:
        sp.install_deps()
        return
    sp.activate_all()
    print('Senpy version {}'.format(senpy.__version__))
    print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                            args.port))
    if not app.debug:
        http_server = HTTPServer(WSGIContainer(app))
        http_server.listen(args.port, address=args.host)
        try:
            IOLoop.instance().start()
        except KeyboardInterrupt:
            print('Bye!')
        http_server.stop()
    else:
        app.run(args.host,
                args.port,
                debug=True)
    sp.deactivate_all()

if __name__ == '__main__':
    main()
<commit_msg>add option to show senpy version number<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#    Copyright 2014 J. Fernando Sánchez Rada - Grupo de Sistemas Inteligentes
#                                                       DIT, UPM
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
"""
Senpy is a modular sentiment analysis server. This script runs an instance of
the server.

"""

from flask import Flask
from senpy.extensions import Senpy
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

import logging
import os
import argparse
import senpy

SERVER_PORT = os.environ.get("PORT", 5000)

def main():
    parser = argparse.ArgumentParser(description='Run a Senpy server')
    parser.add_argument(
        '--level',
        '-l',
        metavar='logging_level',
        type=str,
        default="INFO",
        help='Logging level')
    parser.add_argument(
        '--debug',
        '-d',
        action='store_true',
        default=False,
        help='Run the application in debug mode')
    parser.add_argument(
        '--default-plugins',
        action='store_true',
        default=False,
        help='Load the default plugins')
    parser.add_argument(
        '--host',
        type=str,
        default="0.0.0.0",
        help='Use 0.0.0.0 to accept requests from any host.')
    parser.add_argument(
        '--port',
        '-p',
        type=int,
        default=SERVER_PORT,
        help='Port to listen on.')
    parser.add_argument(
        '--plugins-folder',
        '-f',
        type=str,
        default='plugins',
        help='Where to look for plugins.')
    parser.add_argument(
        '--only-install',
        '-i',
        action='store_true',
        default=False,
        help='Do not run a server, only install plugin dependencies')
    parser.add_argument(
        '--version',
        '-v',
        action='store_true',
        default=False,
        help='Output the senpy version and exit')
    args = parser.parse_args()
    if args.version:
        print('Senpy version {}'.format(senpy.__version__))
        exit(1)
    logging.basicConfig()
    rl = logging.getLogger()
    rl.setLevel(getattr(logging, args.level))
    app = Flask(__name__)
    app.debug = args.debug
    sp = Senpy(app, args.plugins_folder, default_plugins=args.default_plugins)
    if args.only_install:
        sp.install_deps()
        return
    sp.activate_all()
    print('Senpy version {}'.format(senpy.__version__))
    print('Server running on port %s:%d. Ctrl+C to quit' % (args.host,
                                                            args.port))
    if not app.debug:
        http_server = HTTPServer(WSGIContainer(app))
        http_server.listen(args.port, address=args.host)
        try:
            IOLoop.instance().start()
        except KeyboardInterrupt:
            print('Bye!')
        http_server.stop()
    else:
        app.run(args.host,
                args.port,
                debug=True)
    sp.deactivate_all()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
"""
This page is in the table of contents.
==Overview==
===Introduction===
Cura is a AGPL tool chain to generate a GCode path for 3D printing. Older versions of Cura where based on Skeinforge.
Versions up from 13.05 are based on a C++ engine called CuraEngine.
"""
__copyright__ = "Copyright (C) 2013 David Braam - Released under terms of the AGPLv3 License"

from optparse import OptionParser

from Cura.util import profile

def main():
	"""
	Main Cura entry point. Parses arguments, and starts GUI or slicing process depending on the arguments.
	"""
	parser = OptionParser(usage="usage: %prog [options] <filename>.stl")
	parser.add_option("-i", "--ini", action="store", type="string", dest="profileini",
		help="Load settings from a profile ini file")
	parser.add_option("-r", "--print", action="store", type="string", dest="printfile",
		help="Open the printing interface, instead of the normal cura interface.")
	parser.add_option("-p", "--profile", action="store", type="string", dest="profile",
		help="Internal option, do not use!")
	parser.add_option("-s", "--slice", action="store_true", dest="slice",
		help="Slice the given files instead of opening them in Cura")
	parser.add_option("-o", "--output", action="store", type="string", dest="output",
		help="path to write sliced file to")
	parser.add_option("--serialCommunication", action="store", type="string", dest="serialCommunication",
		help="Start commandline serial monitor")

	(options, args) = parser.parse_args()

	if options.serialCommunication:
		from Cura import serialCommunication
		port, baud = options.serialCommunication.split(':')
		serialCommunication.startMonitor(port, baud)
		return

	print "load preferences from " + profile.getPreferencePath()
	profile.loadPreferences(profile.getPreferencePath())

	if options.profile is not None:
		profile.setProfileFromString(options.profile)
	elif options.profileini is not None:
		profile.loadProfile(options.profileini)
	else:
		profile.loadProfile(profile.getDefaultProfilePath(), True)

	if options.printfile is not None:
		from Cura.gui import printWindow
		printWindow.startPrintInterface(options.printfile)
	elif options.slice is not None:
		from Cura.util import sliceEngine
		from Cura.util import objectScene
		from Cura.util import meshLoader
		import shutil

		def commandlineProgressCallback(progress):
			if progress >= 0:
				#print 'Preparing: %d%%' % (progress * 100)
				pass
		scene = objectScene.Scene()
		scene.updateMachineDimensions()
		engine = sliceEngine.Engine(commandlineProgressCallback)
		for m in meshLoader.loadMeshes(args[0]):
			scene.add(m)
		engine.runEngine(scene)
		engine.wait()

		if not options.output:
			options.output = args[0] + profile.getGCodeExtension()
		with open(options.output, "wb") as f:
			gcode = engine.getResult().getGCode()
			while True:
				data = gcode.read()
				if len(data) == 0:
					break
				f.write(data)
		print 'GCode file saved : %s' % options.output

		engine.cleanup()
	else:
		from Cura.gui import app
		app.CuraApp(args).MainLoop()

if __name__ == '__main__':
	main()
<commit_msg>Remove useless -r option<commit_after>#!/usr/bin/python
"""
This page is in the table of contents.
==Overview==
===Introduction===
Cura is a AGPL tool chain to generate a GCode path for 3D printing. Older versions of Cura where based on Skeinforge.
Versions up from 13.05 are based on a C++ engine called CuraEngine.
"""
__copyright__ = "Copyright (C) 2013 David Braam - Released under terms of the AGPLv3 License"

from optparse import OptionParser

from Cura.util import profile

def main():
	"""
	Main Cura entry point. Parses arguments, and starts GUI or slicing process depending on the arguments.
	"""
	parser = OptionParser(usage="usage: %prog [options] <filename>.stl")
	parser.add_option("-i", "--ini", action="store", type="string", dest="profileini",
		help="Load settings from a profile ini file")
	parser.add_option("-p", "--profile", action="store", type="string", dest="profile",
		help="Internal option, do not use!")
	parser.add_option("-s", "--slice", action="store_true", dest="slice",
		help="Slice the given files instead of opening them in Cura")
	parser.add_option("-o", "--output", action="store", type="string", dest="output",
		help="path to write sliced file to")
	parser.add_option("--serialCommunication", action="store", type="string", dest="serialCommunication",
		help="Start commandline serial monitor")

	(options, args) = parser.parse_args()

	if options.serialCommunication:
		from Cura import serialCommunication
		port, baud = options.serialCommunication.split(':')
		serialCommunication.startMonitor(port, baud)
		return

	print "load preferences from " + profile.getPreferencePath()
	profile.loadPreferences(profile.getPreferencePath())

	if options.profile is not None:
		profile.setProfileFromString(options.profile)
	elif options.profileini is not None:
		profile.loadProfile(options.profileini)
	else:
		profile.loadProfile(profile.getDefaultProfilePath(), True)

	if options.slice is not None:
		from Cura.util import sliceEngine
		from Cura.util import objectScene
		from Cura.util import meshLoader
		import shutil

		def commandlineProgressCallback(progress):
			if progress >= 0:
				#print 'Preparing: %d%%' % (progress * 100)
				pass
		scene = objectScene.Scene()
		scene.updateMachineDimensions()
		engine = sliceEngine.Engine(commandlineProgressCallback)
		for m in meshLoader.loadMeshes(args[0]):
			scene.add(m)
		engine.runEngine(scene)
		engine.wait()

		if not options.output:
			options.output = args[0] + profile.getGCodeExtension()
		with open(options.output, "wb") as f:
			gcode = engine.getResult().getGCode()
			while True:
				data = gcode.read()
				if len(data) == 0:
					break
				f.write(data)
		print 'GCode file saved : %s' % options.output

		engine.cleanup()
	else:
		from Cura.gui import app
		app.CuraApp(args).MainLoop()

if __name__ == '__main__':
	main()
<|endoftext|>


<commit_before>import json
import os
from urlparse import urlparse
from csv import DictReader, Sniffer
from StringIO import StringIO
from operator import itemgetter
from itertools import groupby
from time import sleep
from json import dumps

from requests import get
from boto.s3.connection import S3Connection
from boto.s3.key import Key

from app import db, Project

BUCKET = os.environ['S3_BUCKET']
gdocs_url = 'https://docs.google.com/a/codeforamerica.org/spreadsheet/ccc?key=0ArHmv-6U1drqdGNCLWV5Q0d5YmllUzE5WGlUY3hhT2c&output=csv'

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

def get_github_api(url):
    '''
    '''
    got = get(url, auth=github_auth)

    if github_auth is None:
        sleep(1) # be nice to Github

    return got

def get_organizations():
    ''' Get a row for each organization from the Brigade Info spreadsheet.

        Return a list of dictionaries, one for each row past the header.
    '''
    got = get(gdocs_url)
    organizations = list(DictReader(StringIO(got.text)))

    return organizations

def load_projects(projects_list_url):
    ''' Load a list of projects from a given URL.
    '''
    print projects_list_url
    got = get(projects_list_url)

    try:
        projects_details = [dict(code_url=item) for item in got.json()]

    except ValueError:
        data = got.text.splitlines()
        dialect = Sniffer().sniff(data[0])
        projects_details = list(DictReader(data, dialect=dialect))

    return [update_project_info(project_detail) for project_detail in projects_details]

def update_project_info(project_detail):
    ''' Update info from Github, if it's missing.

        Modify the project_detail in-place with new info and return it.

        Complete repository project details go into extras, for example
        project details from Github can be found under "github_extras".
    '''
    if 'code_url' not in project_detail:
        return project_detail

    _, host, path, _, _, _ = urlparse(project_detail['code_url'])

    if host == 'github.com':
        repo_url = 'https://api.github.com/repos' + path

        print repo_url
        got = get_github_api(repo_url)

        if got.status_code in range(400, 499):
            raise IOError('We done got throttled')

        github_project_info = got.json()

        if 'name' not in project_detail or not project_detail['name']:
            project_detail['name'] = github_project_info['name']

        if 'description' not in project_detail or not project_detail['description']:
            project_detail['description'] = github_project_info['description']

        if 'link_url' not in project_detail or not project_detail['link_url']:
            project_detail['link_url'] = github_project_info['homepage']

        project_detail['github_extras'] = unicode(github_project_info)

    return project_detail

def reformat_project_info(input):
    ''' Return a clone of the project hash, formatted for use by opengovhacknight.org.

        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()

    for field in ('name', 'description'):
        output[field] = input[field]

    if 'github_extras' in input:
        try:
            info = collect_github_project_info(input)
        except:
            pass
        else:
            output.update(info)

    return output

def collect_github_project_info(input):
    ''' Collect Github project info, formatted for use by opengovhacknight.org.

        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()
    github = input['github_extras']

    #
    # Populate core Github fields.
    #
    for field in (
            'contributors_url', 'created_at', 'forks_count', 'homepage',
            'html_url', 'id', 'language', 'open_issues', 'pushed_at',
            'updated_at', 'watchers_count'
            ):
        output[field] = github[field]

    output['owner'] = dict()

    for field in ('avatar_url', 'html_url', 'login', 'type'):
        output['owner'][field] = github['owner'][field]

    #
    # Populate project contributors from github[contributors_url]
    #
    output['contributors'] = []
    got = get_github_api(github['contributors_url'])

    for contributor in got.json():
        # we don't want people without email addresses?
        if contributor['login'] == 'invalid-email-address':
            break

        output['contributors'].append(dict())

        for field in ('login', 'url', 'avatar_url', 'html_url', 'contributions'):
            output['contributors'][-1][field] = contributor[field]

        output['contributors'][-1]['owner'] \
            = bool(contributor['login'] == output['owner']['login'])

    #
    # Populate project participation from github[url] + "/stats/participation"
    #
    got = get(github['url'] + '/stats/participation', auth=github_auth)
    output['participation'] = got.json()['all']

    #
    # Populate project needs from github[issues_url] (remove "{/number}")
    #
    output['project_needs'] = []
    url = github['issues_url'].replace('{/number}', '')
    got = get(url, auth=github_auth, params=dict(labels='project-needs'))

    for issue in got.json():
        project_need = dict(title=issue['title'], issue_url=issue['html_url'])
        output['project_needs'].append(project_need)

    return output

def upload_json_file(json_data, object_name):
    ''' Upload a named file to S3 BUCKET, return nothing.
    '''
    s3 = S3Connection()
    bucket = s3.get_bucket(BUCKET)

    object = Key(bucket)
    object.key = object_name

    data = dumps(json_data, indent=2)
    args = dict(policy='public-read', headers={'Content-Type': 'application/json'})

    object.set_contents_from_string(data, **args)

def count_people_totals(all_project_details):
    ''' Create a list of people details based on project details.

        Request additional data from Github API for each person.
    '''
    users, contributors = [], []

    for project_details in all_project_details:
        contributors.extend(project_details['contributors'])

    #
    # Sort by login; there will be duplicates!
    #
    contributors.sort(key=itemgetter('login'))

    #
    # Populate users array with groups of contributors.
    #
    for (_, _contributors) in groupby(contributors, key=itemgetter('login')):
        user = dict(contributions=0, repositories=0)

        for contributor in _contributors:
            user['contributions'] += contributor['contributions']
            user['repositories'] += 1

            if 'login' in user:
                continue

            #
            # Populate user hash with Github info, if it hasn't been already.
            #
            got = get_github_api(contributor['url'])
            contributor = got.json()

            for field in (
                    'login', 'avatar_url', 'html_url',
                    'blog', 'company', 'location'
                    ):
                user[field] = contributor.get(field, None)

        users.append(user)

    return users

if __name__ == "__main__":

    db.drop_all()
    db.create_all()

    all_project_details = []

    for organization in get_organizations():
        print organization['name']
        if not organization['projects_list_url']:
            continue

        org_projects_details = load_projects(organization['projects_list_url'])

        for project_details in org_projects_details:
            project = Project(**project_details)
            db.session.add(project)
            db.session.commit()
        #     all_project_details.append(reformat_project_info(project_details))
        #     print dumps(project, indent=2)

    upload_json_file(all_project_details, 'project_details.json')
    people = count_people_totals(all_project_details)
    upload_json_file(people, 'people.json')
<commit_msg>Format of uploaded json files isnt working.<commit_after>import json
import os
from urlparse import urlparse
from csv import DictReader, Sniffer
from StringIO import StringIO
from operator import itemgetter
from itertools import groupby
from time import sleep
from json import dumps

from requests import get
from boto.s3.connection import S3Connection
from boto.s3.key import Key

from app import db, Project

BUCKET = os.environ['S3_BUCKET']
gdocs_url = 'https://docs.google.com/a/codeforamerica.org/spreadsheet/ccc?key=0ArHmv-6U1drqdGNCLWV5Q0d5YmllUzE5WGlUY3hhT2c&output=csv'

if 'GITHUB_TOKEN' in os.environ:
    github_auth = (os.environ['GITHUB_TOKEN'], '')
else:
    github_auth = None

def get_github_api(url):
    '''
    '''
    got = get(url, auth=github_auth)

    if github_auth is None:
        sleep(1) # be nice to Github

    return got

def get_organizations():
    ''' Get a row for each organization from the Brigade Info spreadsheet.

        Return a list of dictionaries, one for each row past the header.
    '''
    got = get(gdocs_url)
    organizations = list(DictReader(StringIO(got.text)))

    return organizations

def load_projects(projects_list_url):
    ''' Load a list of projects from a given URL.
    '''
    print projects_list_url
    got = get(projects_list_url)

    try:
        projects_details = [dict(code_url=item) for item in got.json()]

    except ValueError:
        data = got.text.splitlines()
        dialect = Sniffer().sniff(data[0])
        projects_details = list(DictReader(data, dialect=dialect))

    return [update_project_info(project_detail) for project_detail in projects_details]

def update_project_info(project_detail):
    ''' Update info from Github, if it's missing.

        Modify the project_detail in-place with new info and return it.

        Complete repository project details go into extras, for example
        project details from Github can be found under "github_extras".
    '''
    if 'code_url' not in project_detail:
        return project_detail

    _, host, path, _, _, _ = urlparse(project_detail['code_url'])

    if host == 'github.com':
        repo_url = 'https://api.github.com/repos' + path

        print repo_url
        got = get_github_api(repo_url)

        if got.status_code in range(400, 499):
            raise IOError('We done got throttled')

        github_project_info = got.json()

        if 'name' not in project_detail or not project_detail['name']:
            project_detail['name'] = github_project_info['name']

        if 'description' not in project_detail or not project_detail['description']:
            project_detail['description'] = github_project_info['description']

        if 'link_url' not in project_detail or not project_detail['link_url']:
            project_detail['link_url'] = github_project_info['homepage']

        project_detail['github_extras'] = unicode(github_project_info)

    return project_detail

def reformat_project_info(input):
    ''' Return a clone of the project hash, formatted for use by opengovhacknight.org.

        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()

    for field in ('name', 'description'):
        output[field] = input[field]

    if 'github_extras' in input:
        try:
            info = collect_github_project_info(input)
        except:
            pass
        else:
            output.update(info)

    return output

def collect_github_project_info(input):
    ''' Collect Github project info, formatted for use by opengovhacknight.org.

        The representation here is specifically expected to be used on this page:
        http://opengovhacknight.org/projects.html
    '''
    output = dict()
    github = input['github_extras']

    #
    # Populate core Github fields.
    #
    for field in (
            'contributors_url', 'created_at', 'forks_count', 'homepage',
            'html_url', 'id', 'language', 'open_issues', 'pushed_at',
            'updated_at', 'watchers_count'
            ):
        output[field] = github[field]

    output['owner'] = dict()

    for field in ('avatar_url', 'html_url', 'login', 'type'):
        output['owner'][field] = github['owner'][field]

    #
    # Populate project contributors from github[contributors_url]
    #
    output['contributors'] = []
    got = get_github_api(github['contributors_url'])

    for contributor in got.json():
        # we don't want people without email addresses?
        if contributor['login'] == 'invalid-email-address':
            break

        output['contributors'].append(dict())

        for field in ('login', 'url', 'avatar_url', 'html_url', 'contributions'):
            output['contributors'][-1][field] = contributor[field]

        output['contributors'][-1]['owner'] \
            = bool(contributor['login'] == output['owner']['login'])

    #
    # Populate project participation from github[url] + "/stats/participation"
    #
    got = get(github['url'] + '/stats/participation', auth=github_auth)
    output['participation'] = got.json()['all']

    #
    # Populate project needs from github[issues_url] (remove "{/number}")
    #
    output['project_needs'] = []
    url = github['issues_url'].replace('{/number}', '')
    got = get(url, auth=github_auth, params=dict(labels='project-needs'))

    for issue in got.json():
        project_need = dict(title=issue['title'], issue_url=issue['html_url'])
        output['project_needs'].append(project_need)

    return output

def upload_json_file(json_data, object_name):
    ''' Upload a named file to S3 BUCKET, return nothing.
    '''
    s3 = S3Connection()
    bucket = s3.get_bucket(BUCKET)

    object = Key(bucket)
    object.key = object_name

    data = dumps(json_data, indent=2)
    args = dict(policy='public-read', headers={'Content-Type': 'application/json'})

    object.set_contents_from_string(data, **args)

def count_people_totals(all_project_details):
    ''' Create a list of people details based on project details.

        Request additional data from Github API for each person.
    '''
    users, contributors = [], []

    for project_details in all_project_details:
        contributors.extend(project_details['contributors'])

    #
    # Sort by login; there will be duplicates!
    #
    contributors.sort(key=itemgetter('login'))

    #
    # Populate users array with groups of contributors.
    #
    for (_, _contributors) in groupby(contributors, key=itemgetter('login')):
        user = dict(contributions=0, repositories=0)

        for contributor in _contributors:
            user['contributions'] += contributor['contributions']
            user['repositories'] += 1

            if 'login' in user:
                continue

            #
            # Populate user hash with Github info, if it hasn't been already.
            #
            got = get_github_api(contributor['url'])
            contributor = got.json()

            for field in (
                    'login', 'avatar_url', 'html_url',
                    'blog', 'company', 'location'
                    ):
                user[field] = contributor.get(field, None)

        users.append(user)

    return users

if __name__ == "__main__":

    db.drop_all()
    db.create_all()

    all_project_details = []

    for organization in get_organizations():
        print organization['name']
        if not organization['projects_list_url']:
            continue

        org_projects_details = load_projects(organization['projects_list_url'])

        for project_details in org_projects_details:
            project = Project(**project_details)
            db.session.add(project)
            db.session.commit()
            # all_project_details.append(reformat_project_info(project_details))
            # print dumps(project, indent=2)

    upload_json_file(all_project_details, 'project_details.json')
    people = count_people_totals(all_project_details)
    upload_json_file(people, 'people.json')
<|endoftext|>


<commit_before><commit_msg>Fix logg calcs, tidy up (not fix!) T1 & T2 calcs, add makePlots flag<commit_after><|endoftext|>