<commit_before>from django.test import TestCase
from .stocktwits import get_stock_comments, format_into_table, save_message
from .reddit import (
    get_companies,
    ticker_to_name,
    scrape_reddit,
    save_reddit_articles
)
from .models import Message
import requests
import datetime

# Create your tests here.

EXAMPLE_RESPONSE = {
    'body': 'What This 'Esteemed' Venture Capitalist Learned From Mark Zuckerberg $FB $MSFT $YHOO http://stkw.it/d2Ub',
    'symbols': [
        {'symbol': 'MSFT', 'id': 2735, 'title': 'Microsoft Corporation'},
        {'symbol': 'YHOO', 'id': 4177, 'title': 'Yahoo! Inc.'},
        {'symbol': 'FB', 'id': 7871, 'title': 'Facebook'}
    ], 'user': {
        'identity': 'Official',
        'classification': ['suggested', 'official'],
        'avatar_url_ssl': 'https://s3.amazonaws.com/st-avatars/production/7108/thumb-1301323720.png',
        'id': 7108,
        'name': 'Benzinga',
        'avatar_url': 'http://avatars.stocktwits.com/production/7108/thumb-1301323720.png',
        'official': True,
        'join_date': '2009-12-04',
        'username': 'Benzinga'
    },
    'id': 51852548,
    'links': [
        {
            'source': {
                'website': 'http://www.benzinga.com',
                'name': 'Benzinga'
            },
            'image': 'http://cdn3.benzinga.com/files/imagecache/story_image_685x375C/images/story/2012/2321206299_5038c90c4e_o.jpg',
            'video_url': None,
            'url': 'http://www.benzinga.com/general/entrepreneurship/16/03/7765501/what-this-esteemed-venture-capitalist-learned-from-mark-zucke',
            'description': 'Chamath Palihapitiya runs one of the best-respected venture capital firms in Silicon Valley, Social Capital, which has backed several tech companies with valuations in the billions, including Slack, Box and SurveyMonkey. Vanity Fair recently interviewed Palihapitiya. The former Facebook Inc (NASDAQ: FB) employee went into multiple issues, and this article will focus on what the investor learned from working side-by-side with Mark Zuckerberg.',
            'shortened_url': 'http://stkw.it/d2Ub',
            'title': "Facebook, Inc. (NASDAQ:FB), Microsoft Corporation (NASDAQ:MSFT) - What This 'Esteemed' Venture Capitalist Learned From Mark Zuckerberg",
            'created_at': '2016-03-28T21:51:09Z', 'shortened_expanded_url': 'benzinga.com/general/entrep...'
        }
    ], 'source': {
        'title': 'StockTwits',
        'id': 1,
        'url': 'http://stocktwits.com'
    },
    'created_at': '2016-03-28T21:51:06Z',
    'reshares': {'reshared_count': 0, 'user_ids': []}
}

EXPECTED = {
    "social_id": "51852548",
    "source": "stocktwits",
    "focus": "MSFT",
    "popularity": 0,
    "author": "Benzinga",
    "author_image": "https://s3.amazonaws.com/st-avatars/production/7108/thumb-1301323720.png",
    "created_time": '2016-03-28T21:51:06Z',
    "content": 'What This \'Esteemed\' Venture Capitalist Learned From Mark Zuckerberg $FB $MSFT $YHOO http://stkw.it/d2Ub',
    "symbols": ['MSFT', 'YHOO', 'FB'],
    "urls": ['http://www.benzinga.com/general/entrepreneurship/16/03/7765501/what-this-esteemed-venture-capitalist-learned-from-mark-zucke'],
    "url": 'http://stocktwits.com/Benzinga/message/51852548'
}

class RedditScraper(TestCase):

    def test_json_loads(self):
        companies = get_companies()
        expected = {
            "Ticker": "MSFT",
            "Name": "Microsoft Corporation",
            "Exchange": "NMS",
            "Country": "USA",
            "Category Name": "Business Software & Services",
            "Category Number": 826,
            "": 0
        }
        self.assertEqual(companies['MSFT'], expected)

    def test_ticker_to_name_works(self):
        companies = get_companies()
        expected = "Microsoft Corporation"
        self.assertEqual(ticker_to_name(companies, "MSFT"), expected)

    def test_ticker_to_name_bad_company_data(self):
        with self.assertRaises(ValueError):
            ticker_to_name("waffles", "MSFT")

    def test_ticker_to_name_bad_ticker(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, "ayyooo")

    def test_reddit_scraper(self):
        links = scrape_reddit("MSFT", "Microsoft Corporation")
        expected = 'https://www.reddit.com/r/linux/comments/mgtht/given_the_recent_protests_shouldnt_we_point_out/'
        self.assertIn(expected, str(links))

    def test_reddit_save(self):
        links = scrape_reddit("AAPL", ticker_to_name(get_companies(), "SUNE"))
        expected = {
            'url': 'http://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/?ref=search_posts',
            'urls': ['https://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/'],
            'social_id': '40wx7b',
            'created_time': datetime.datetime.utcfromtimestamp(1452764067.0).strftime("%Y-%m-%dT%H:%M:%SZ"),
            'content': 'Sunedison Inc To Distribute Tesla Motors Inc Powerwall',
            'author': 'MartEden',
            'popularity': 27,
            'source': 'reddit',
            'author_image': 'https://www.redditstatic.com/icon-touch.png',
            'focus': 'AAPL',
            'symbols': ['AAPL']
        }
        save_reddit_articles(links)
        dbobj = Message.objects.get(social_id=expected['social_id'])
        self.assertEqual(dbobj.url, expected['url'])
        self.assertEqual(dbobj.urls, str(expected['urls']))
        self.assertEqual(dbobj.content, expected['content'])
        self.assertEqual(dbobj.author, expected['author'])
        self.assertEqual(dbobj.focus, 'AAPL')

class StockTwitsCase(TestCase):

    def test_error_is_raised(self):
        with self.assertRaises(ValueError):
            get_stock_comments("not a real company")

    def test_response_length(self):
        response = get_stock_comments("MSFT")
        self.assertEqual(len(response), 30)

    def test_response_contains_id(self):
        response = get_stock_comments("MSFT")
        for r in response:
            self.assertNotEqual(r.get('id'), None)

    def test_response_contains_body(self):
        response = get_stock_comments("MSFT")
        for r in response:
            self.assertNotEqual(r.get('body'), None)

    def test_json_trimmer(self):
        formatted = format_into_table(EXAMPLE_RESPONSE, "MSFT")
        self.assertEqual(formatted, EXPECTED)

    def test_database_save_return(self):
        self.assertTrue(save_message(EXPECTED))
        self.assertFalse(save_message(EXPECTED))

    def test_database_save_database(self):
        save_message(EXPECTED)
        m = Message.objects.get(social_id="51852548")
        self.assertEqual(m.focus, "MSFT")
        self.assertEqual(m.url, "http://stocktwits.com/Benzinga/message/51852548")

    def test_text_is_correct(self):
        save_message(EXPECTED)
        message = Message.objects.get(social_id="51852548")
        document = requests.get(message.url).text
        self.assertIn(message.content, document)
<commit_msg>Created ticker tests<commit_after>from django.test import TestCase, Client
from .stocktwits import get_stock_comments, format_into_table, save_message
from .reddit import (
    get_companies,
    ticker_to_name,
    scrape_reddit,
    save_reddit_articles
)
from .models import Message
import requests
import datetime

# Create your tests here.

EXAMPLE_RESPONSE = {
    'body': 'What This 'Esteemed' Venture Capitalist Learned From Mark Zuckerberg $FB $MSFT $YHOO http://stkw.it/d2Ub',
    'symbols': [
        {'symbol': 'MSFT', 'id': 2735, 'title': 'Microsoft Corporation'},
        {'symbol': 'YHOO', 'id': 4177, 'title': 'Yahoo! Inc.'},
        {'symbol': 'FB', 'id': 7871, 'title': 'Facebook'}
    ], 'user': {
        'identity': 'Official',
        'classification': ['suggested', 'official'],
        'avatar_url_ssl': 'https://s3.amazonaws.com/st-avatars/production/7108/thumb-1301323720.png',
        'id': 7108,
        'name': 'Benzinga',
        'avatar_url': 'http://avatars.stocktwits.com/production/7108/thumb-1301323720.png',
        'official': True,
        'join_date': '2009-12-04',
        'username': 'Benzinga'
    },
    'id': 51852548,
    'links': [
        {
            'source': {
                'website': 'http://www.benzinga.com',
                'name': 'Benzinga'
            },
            'image': 'http://cdn3.benzinga.com/files/imagecache/story_image_685x375C/images/story/2012/2321206299_5038c90c4e_o.jpg',
            'video_url': None,
            'url': 'http://www.benzinga.com/general/entrepreneurship/16/03/7765501/what-this-esteemed-venture-capitalist-learned-from-mark-zucke',
            'description': 'Chamath Palihapitiya runs one of the best-respected venture capital firms in Silicon Valley, Social Capital, which has backed several tech companies with valuations in the billions, including Slack, Box and SurveyMonkey. Vanity Fair recently interviewed Palihapitiya. The former Facebook Inc (NASDAQ: FB) employee went into multiple issues, and this article will focus on what the investor learned from working side-by-side with Mark Zuckerberg.',
            'shortened_url': 'http://stkw.it/d2Ub',
            'title': "Facebook, Inc. (NASDAQ:FB), Microsoft Corporation (NASDAQ:MSFT) - What This 'Esteemed' Venture Capitalist Learned From Mark Zuckerberg",
            'created_at': '2016-03-28T21:51:09Z', 'shortened_expanded_url': 'benzinga.com/general/entrep...'
        }
    ], 'source': {
        'title': 'StockTwits',
        'id': 1,
        'url': 'http://stocktwits.com'
    },
    'created_at': '2016-03-28T21:51:06Z',
    'reshares': {'reshared_count': 0, 'user_ids': []}
}

EXPECTED = {
    "social_id": "51852548",
    "source": "stocktwits",
    "focus": "MSFT",
    "popularity": 0,
    "author": "Benzinga",
    "author_image": "https://s3.amazonaws.com/st-avatars/production/7108/thumb-1301323720.png",
    "created_time": '2016-03-28T21:51:06Z',
    "content": 'What This \'Esteemed\' Venture Capitalist Learned From Mark Zuckerberg $FB $MSFT $YHOO http://stkw.it/d2Ub',
    "symbols": ['MSFT', 'YHOO', 'FB'],
    "urls": ['http://www.benzinga.com/general/entrepreneurship/16/03/7765501/what-this-esteemed-venture-capitalist-learned-from-mark-zucke'],
    "url": 'http://stocktwits.com/Benzinga/message/51852548'
}

class TickerTest(TestCase):

    def test_check_view_status_code(self):
        client = Client()
        response = client.get('/check/msft/')
        self.assertEqual(response.status_code, 200)

    def test_check_view_content(self):
        client = Client()
        response = client.get('/check/msft/')
        json_blob = response.json()
        self.assertTrue(isinstance(json_blob, list))
        self.assertTrue(isinstance(json_blob[0], dict))
        self.assertIn('author', json_blob[0])
        self.assertIn('content', json_blob[0])
        self.assertIn('social_id', json_blob[0])
        self.assertIn('url', json_blob[0])
        self.assertIn('urls', json_blob[0])
        self.assertIn('popularity', json_blob[0])
        self.assertIn('source', json_blob[0])
        self.assertIn('created_time', json_blob[0])
        self.assertIn('symbols', json_blob[0])
        self.assertIn('focus', json_blob[0])
        self.assertIn('author_image', json_blob[0])

class RedditScraper(TestCase):

    def test_json_loads(self):
        companies = get_companies()
        expected = {
            "Ticker": "MSFT",
            "Name": "Microsoft Corporation",
            "Exchange": "NMS",
            "Country": "USA",
            "Category Name": "Business Software & Services",
            "Category Number": 826,
            "": 0
        }
        self.assertEqual(companies['MSFT'], expected)

    def test_ticker_to_name_works(self):
        companies = get_companies()
        expected = "Microsoft Corporation"
        self.assertEqual(ticker_to_name(companies, "MSFT"), expected)

    def test_ticker_to_name_bad_company_data(self):
        with self.assertRaises(ValueError):
            ticker_to_name("waffles", "MSFT")

    def test_ticker_to_name_bad_ticker(self):
        companies = get_companies()
        with self.assertRaises(ValueError):
            ticker_to_name(companies, "ayyooo")

    def test_reddit_scraper(self):
        links = scrape_reddit("MSFT", "Microsoft Corporation")
        expected = 'https://www.reddit.com/r/linux/comments/mgtht/given_the_recent_protests_shouldnt_we_point_out/'
        self.assertIn(expected, str(links))

    def test_reddit_save(self):
        links = scrape_reddit("AAPL", ticker_to_name(get_companies(), "SUNE"))
        expected = {
            'url': 'http://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/?ref=search_posts',
            'urls': ['https://www.reddit.com/r/investing/comments/40wx7b/sunedison_inc_to_distribute_tesla_motors_inc/'],
            'social_id': '40wx7b',
            'created_time': datetime.datetime.utcfromtimestamp(1452764067.0).strftime("%Y-%m-%dT%H:%M:%SZ"),
            'content': 'Sunedison Inc To Distribute Tesla Motors Inc Powerwall',
            'author': 'MartEden',
            'popularity': 27,
            'source': 'reddit',
            'author_image': 'https://www.redditstatic.com/icon-touch.png',
            'focus': 'AAPL',
            'symbols': ['AAPL']
        }
        save_reddit_articles(links)
        dbobj = Message.objects.get(social_id=expected['social_id'])
        self.assertEqual(dbobj.url, expected['url'])
        self.assertEqual(dbobj.urls, str(expected['urls']))
        self.assertEqual(dbobj.content, expected['content'])
        self.assertEqual(dbobj.author, expected['author'])
        self.assertEqual(dbobj.focus, 'AAPL')

class StockTwitsCase(TestCase):

    def test_error_is_raised(self):
        with self.assertRaises(ValueError):
            get_stock_comments("not a real company")

    def test_response_length(self):
        response = get_stock_comments("MSFT")
        self.assertEqual(len(response), 30)

    def test_response_contains_id(self):
        response = get_stock_comments("MSFT")
        for r in response:
            self.assertNotEqual(r.get('id'), None)

    def test_response_contains_body(self):
        response = get_stock_comments("MSFT")
        for r in response:
            self.assertNotEqual(r.get('body'), None)

    def test_json_trimmer(self):
        formatted = format_into_table(EXAMPLE_RESPONSE, "MSFT")
        self.assertEqual(formatted, EXPECTED)

    def test_database_save_return(self):
        self.assertTrue(save_message(EXPECTED))
        self.assertFalse(save_message(EXPECTED))

    def test_database_save_database(self):
        save_message(EXPECTED)
        m = Message.objects.get(social_id="51852548")
        self.assertEqual(m.focus, "MSFT")
        self.assertEqual(m.url, "http://stocktwits.com/Benzinga/message/51852548")

    def test_text_is_correct(self):
        save_message(EXPECTED)
        message = Message.objects.get(social_id="51852548")
        document = requests.get(message.url).text
        self.assertIn(message.content, document)
<|endoftext|>


<commit_before>"""
hellofriend.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
<commit_msg>Updated from Brython Server: 9/10/2015 2:32:15 PM<commit_after>"""
hellofriend.py
Author: <your name here>
Credit: <list sources used, if any>

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
user=input("Tell me your name!")<|endoftext|>


<commit_before>"""
hellofriend.py
Author: Sarah Dunbar
Credit: none

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""

name = input("Please tell me your name:")
age = input("Please tell me your age:")
pycomp = 24 - int(age)
print ("Hello, " + name + ". Python is " + str(pycomp) + " years older than you are!")<commit_msg>Updated from Brython Server: 9/8/2015 2:57:15 PM<commit_after>"""
hellofriend.py
Author: Sarah Dunbar
Credit: none

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""

name = input("Please tell me your name: ")
age = input("Please tell me your age: ")
pycomp = 24 - int(age)
print ("Hello, " + name + ". Python is " + str(pycomp) + " years older than you are!")<|endoftext|>


<commit_before>"""
hellofriend.py
Author: <your name here>
Credit: Payton; Payton; Payton some more; and oh wait, more Payton

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
name = input("What is your name?")
age = int(input("How old are you?"))
if(age>25):
    s= "Hello, {0}. Python is {1} years {2} than you are!"
    print(s.format(name, age-25, "younger"))

if(age==25):

if(age<25):

<commit_msg>Updated from Brython Server: 1/22/2016 2:53:59 PM<commit_after>"""
hellofriend.py
Author: Andreas
Credit: Payton; Payton; Payton some more; and oh wait, more Payton; have some more Payton

Assignment:

Write and submit an interactive Python program that asks for the user's name and age,
then prints how much older Python is than the user (based on a simple comparison of
birth year). Python's first public release occurred in 1991. Something like this:

Please tell me your name: Guido
Please tell me your age: 16
Hello, Guido. Python is 8 years older than you are!

Note that the text: "Guido" and "16" are entered by the user running the program.
The final line ("Hello...") is generated dynamically when you run the program, based
on the name and age that the user enters.
"""
name = input("What is your name?")
age = int(input("How old are you?"))
if(age>25):
    s= "Hello, {0}. Python is {1} years {2} than you are!"
    print(s.format(name, age-25, "younger"))

if(age==25):
    q="Hello, {0}. Python is the same age as you are!"
    print(q.format(name))

if(age<25):
    print(s.format(name, 25-age, "older"))

<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import binascii
import os
from io import BytesIO
from operator import attrgetter

import click
from PIL import Image

from indico.modules.events.layout import layout_settings
from indico.modules.events.models.events import Event
from indico.util.console import cformat, verbose_iterator
from indico.util.fs import secure_filename
from indico.util.struct.iterables import committing_iterator
from indico_zodbimport import Importer, convert_to_unicode
from indico_zodbimport.util import get_archived_file

ALLOWED_THEMES = {'orange.css', 'brown.css', 'right_menu.css'}

class EventLayoutImporter(Importer):
    def __init__(self, **kwargs):
        self.archive_dirs = kwargs.pop('archive_dir')
        super(EventLayoutImporter, self).__init__(**kwargs)

    @staticmethod
    def decorate_command(command):
        command = click.option('--archive-dir', required=True, multiple=True,
                               help="The base path where resources are stored (ArchiveDir in indico.conf). "
                                    "When used multiple times, the dirs are checked in order until a file is "
                                    "found.")(command)
        return command

    def has_data(self):
        return bool(layout_settings.query.filter_by(name='is_searchable').count())

    def migrate(self):
        self.migrate_layout_settings()

    def migrate_layout_settings(self):
        print cformat('%{white!}migrating layout settings, event logos and custom CSS templates')

        default_styles = self.zodb_root['MaKaCInfo']['main']._styleMgr._defaultEventStylesheet
        for event, event_type, dmgr, logo, custom_css in committing_iterator(self._iter_event_layout_data()):
            if event_type != 'conference':
                theme = dmgr._defaultstyle
                if not theme or theme == default_styles[event_type]:
                    continue
                layout_settings.set(event, 'timetable_theme', theme)
                if not self.quiet:
                    self.print_success(cformat('- %{cyan}Default timetable theme: {}').format(theme), event_id=event.id)
                continue

            settings = self._get_event_settings(event, dmgr)
            layout_settings.set_multi(event, settings)
            if not self.quiet:
                self.print_success(cformat('- %{cyan}Layout settings'), event_id=event.id)
            if logo or custom_css:
                e = Event.get(event.id)
                if not e:
                    self.print_warning('Event does not exist (anymore)! Logo and/or CSS file not saved!',
                                       event_id=event.id)
                    continue
            if logo:
                path = get_archived_file(logo, self.archive_dirs)[1]
                if path is None:
                    self.print_error(cformat('%{red!}{} logo not found on disk; skipping it').format(event),
                                     event_id=event.id)
                    continue
                try:
                    logo_image = Image.open(path)
                except IOError as e:
                    self.print_warning('Cannot open the file. The file probably is not an image.')
                    self.print_warning("Cannot open {}: {}".format(path, e))
                    logo_image = None

                if logo_image:
                    logo_bytes = BytesIO()
                    logo_image.save(logo_bytes, 'PNG')
                    logo_bytes.seek(0)
                    logo_content = logo_bytes.read()
                    logo_filename = secure_filename(convert_to_unicode(logo.fileName), 'logo')
                    logo_filename = os.path.splitext(logo_filename)[0] + '.png'
                    logo_metadata = {
                        'size': len(logo_content),
                        'hash': binascii.crc32(logo_content) & 0xffffffff,
                        'filename': logo_filename,
                        'content_type': 'image/png'
                    }
                    e.logo = logo_content
                    e.logo_metadata = logo_metadata
                    if not self.quiet:
                        self.print_success(cformat('- %{cyan}[Logo] {}').format(logo.fileName), event_id=event.id)
            if custom_css:
                stylesheet = custom_css._localFile
                path = get_archived_file(stylesheet, self.archive_dirs)[1]
                if path is None:
                    self.print_error(cformat('%{red!}{} CSS file not found on disk; skipping it').format(event),
                                     event_id=event.id)
                    continue
                with open(path, 'rb') as f:
                    stylesheet_content = convert_to_unicode(f.read())
                e.stylesheet_metadata = {
                    'size': len(stylesheet_content),
                    'hash': binascii.crc32(stylesheet_content) & 0xffffffff,
                    'filename': secure_filename(convert_to_unicode(stylesheet.fileName), 'stylesheet.css'),
                    'content_type': 'text/css'
                }
                e.stylesheet = stylesheet_content
                if not self.quiet:
                    self.print_success(cformat('- %{cyan}[CSS] {}').format(stylesheet.fileName), event_id=event.id)

    def _iter_event_layout_data(self):
        it = self.zodb_root['conferences'].itervalues()
        if self.quiet:
            it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title'))
        wfr = self.zodb_root['webfactoryregistry']
        dmr = self.zodb_root['displayRegistery']
        for event in self.flushing_iterator(it):
            wf = wfr.get(event.id)
            event_type = 'conference' if wf is None else wf.id
            try:
                dmgr = dmr[event.id]
            except KeyError:
                self.print_error('Skipping event with no displaymgr', event_id=event_id)
                continue
            style_mgr = getattr(dmgr, '_styleMngr', None) if event_type == 'conference' else None
            custom_css = getattr(style_mgr, '_css', None) if event_type == 'conference' else None
            yield event, event_type, dmgr, event._logo, custom_css

    def _get_event_settings(self, event, dmgr):
        format_opts = getattr(dmgr, '_format', None)
        tt = getattr(dmgr, '_tickerTape', None)
        style_mgr = getattr(dmgr, '_styleMngr', None)
        event_id = event.id
        menu = getattr(dmgr, '_menu', None)

        settings = {
            'is_searchable': getattr(dmgr, '_searchEnabled', None),
            'show_nav_bar': getattr(dmgr, '_displayNavigationBar', None),
            'show_social_badges': getattr(dmgr, '_showSocialApps', None),
        }
        if format_opts:
            settings['header_text_color'] = format_opts._data.get('titleTextColor')
            settings['header_background_color'] = format_opts._data.get('titleBgColor')
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _format attribute'), event_id=event_id)
        if tt:
            settings['show_banner'] = getattr(tt, '_enabledNowPlaying', None)
            settings['announcement'] = getattr(tt, '_text', None)
            settings['show_announcement'] = getattr(tt, '_enabledSimpleText', None)
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _tickerTape attribute'),
                             event_id=event_id)
        if style_mgr:
            template = getattr(style_mgr, '_usingTemplate', None)
            theme = getattr(template, 'templateId', None)
            settings['theme'] = theme if theme in ALLOWED_THEMES else None
            settings['use_custom_css'] = getattr(style_mgr, '_css', None) is not None
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _styleMngr attribute'), event_id=event_id)
        if menu:
            settings['timetable_by_room'] = getattr(menu, '_timetable_layout', None) == 'room'
            settings['timetable_detailed'] = getattr(menu, '_timetable_detailed_view', False)
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _menu attribute'), event_id=event_id)

        return {k: v for k, v in settings.iteritems() if v is not None}
<commit_msg>Fix invalid variable<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import binascii
import os
from io import BytesIO
from operator import attrgetter

import click
from PIL import Image

from indico.modules.events.layout import layout_settings
from indico.modules.events.models.events import Event
from indico.util.console import cformat, verbose_iterator
from indico.util.fs import secure_filename
from indico.util.struct.iterables import committing_iterator
from indico_zodbimport import Importer, convert_to_unicode
from indico_zodbimport.util import get_archived_file

ALLOWED_THEMES = {'orange.css', 'brown.css', 'right_menu.css'}

class EventLayoutImporter(Importer):
    def __init__(self, **kwargs):
        self.archive_dirs = kwargs.pop('archive_dir')
        super(EventLayoutImporter, self).__init__(**kwargs)

    @staticmethod
    def decorate_command(command):
        command = click.option('--archive-dir', required=True, multiple=True,
                               help="The base path where resources are stored (ArchiveDir in indico.conf). "
                                    "When used multiple times, the dirs are checked in order until a file is "
                                    "found.")(command)
        return command

    def has_data(self):
        return bool(layout_settings.query.filter_by(name='is_searchable').count())

    def migrate(self):
        self.migrate_layout_settings()

    def migrate_layout_settings(self):
        print cformat('%{white!}migrating layout settings, event logos and custom CSS templates')

        default_styles = self.zodb_root['MaKaCInfo']['main']._styleMgr._defaultEventStylesheet
        for event, event_type, dmgr, logo, custom_css in committing_iterator(self._iter_event_layout_data()):
            if event_type != 'conference':
                theme = dmgr._defaultstyle
                if not theme or theme == default_styles[event_type]:
                    continue
                layout_settings.set(event, 'timetable_theme', theme)
                if not self.quiet:
                    self.print_success(cformat('- %{cyan}Default timetable theme: {}').format(theme), event_id=event.id)
                continue

            settings = self._get_event_settings(event, dmgr)
            layout_settings.set_multi(event, settings)
            if not self.quiet:
                self.print_success(cformat('- %{cyan}Layout settings'), event_id=event.id)
            if logo or custom_css:
                e = Event.get(event.id)
                if not e:
                    self.print_warning('Event does not exist (anymore)! Logo and/or CSS file not saved!',
                                       event_id=event.id)
                    continue
            if logo:
                path = get_archived_file(logo, self.archive_dirs)[1]
                if path is None:
                    self.print_error(cformat('%{red!}{} logo not found on disk; skipping it').format(event),
                                     event_id=event.id)
                    continue
                try:
                    logo_image = Image.open(path)
                except IOError as e:
                    self.print_warning('Cannot open the file. The file probably is not an image.')
                    self.print_warning("Cannot open {}: {}".format(path, e))
                    logo_image = None

                if logo_image:
                    logo_bytes = BytesIO()
                    logo_image.save(logo_bytes, 'PNG')
                    logo_bytes.seek(0)
                    logo_content = logo_bytes.read()
                    logo_filename = secure_filename(convert_to_unicode(logo.fileName), 'logo')
                    logo_filename = os.path.splitext(logo_filename)[0] + '.png'
                    logo_metadata = {
                        'size': len(logo_content),
                        'hash': binascii.crc32(logo_content) & 0xffffffff,
                        'filename': logo_filename,
                        'content_type': 'image/png'
                    }
                    e.logo = logo_content
                    e.logo_metadata = logo_metadata
                    if not self.quiet:
                        self.print_success(cformat('- %{cyan}[Logo] {}').format(logo.fileName), event_id=event.id)
            if custom_css:
                stylesheet = custom_css._localFile
                path = get_archived_file(stylesheet, self.archive_dirs)[1]
                if path is None:
                    self.print_error(cformat('%{red!}{} CSS file not found on disk; skipping it').format(event),
                                     event_id=event.id)
                    continue
                with open(path, 'rb') as f:
                    stylesheet_content = convert_to_unicode(f.read())
                e.stylesheet_metadata = {
                    'size': len(stylesheet_content),
                    'hash': binascii.crc32(stylesheet_content) & 0xffffffff,
                    'filename': secure_filename(convert_to_unicode(stylesheet.fileName), 'stylesheet.css'),
                    'content_type': 'text/css'
                }
                e.stylesheet = stylesheet_content
                if not self.quiet:
                    self.print_success(cformat('- %{cyan}[CSS] {}').format(stylesheet.fileName), event_id=event.id)

    def _iter_event_layout_data(self):
        it = self.zodb_root['conferences'].itervalues()
        if self.quiet:
            it = verbose_iterator(it, len(self.zodb_root['conferences']), attrgetter('id'), attrgetter('title'))
        wfr = self.zodb_root['webfactoryregistry']
        dmr = self.zodb_root['displayRegistery']
        for event in self.flushing_iterator(it):
            wf = wfr.get(event.id)
            event_type = 'conference' if wf is None else wf.id
            try:
                dmgr = dmr[event.id]
            except KeyError:
                self.print_error('Skipping event with no displaymgr', event_id=event.id)
                continue
            style_mgr = getattr(dmgr, '_styleMngr', None) if event_type == 'conference' else None
            custom_css = getattr(style_mgr, '_css', None) if event_type == 'conference' else None
            yield event, event_type, dmgr, event._logo, custom_css

    def _get_event_settings(self, event, dmgr):
        format_opts = getattr(dmgr, '_format', None)
        tt = getattr(dmgr, '_tickerTape', None)
        style_mgr = getattr(dmgr, '_styleMngr', None)
        event_id = event.id
        menu = getattr(dmgr, '_menu', None)

        settings = {
            'is_searchable': getattr(dmgr, '_searchEnabled', None),
            'show_nav_bar': getattr(dmgr, '_displayNavigationBar', None),
            'show_social_badges': getattr(dmgr, '_showSocialApps', None),
        }
        if format_opts:
            settings['header_text_color'] = format_opts._data.get('titleTextColor')
            settings['header_background_color'] = format_opts._data.get('titleBgColor')
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _format attribute'), event_id=event_id)
        if tt:
            settings['show_banner'] = getattr(tt, '_enabledNowPlaying', None)
            settings['announcement'] = getattr(tt, '_text', None)
            settings['show_announcement'] = getattr(tt, '_enabledSimpleText', None)
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _tickerTape attribute'),
                             event_id=event_id)
        if style_mgr:
            template = getattr(style_mgr, '_usingTemplate', None)
            theme = getattr(template, 'templateId', None)
            settings['theme'] = theme if theme in ALLOWED_THEMES else None
            settings['use_custom_css'] = getattr(style_mgr, '_css', None) is not None
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _styleMngr attribute'), event_id=event_id)
        if menu:
            settings['timetable_by_room'] = getattr(menu, '_timetable_layout', None) == 'room'
            settings['timetable_detailed'] = getattr(menu, '_timetable_detailed_view', False)
        else:
            self.print_error(cformat('%{red!} Skipping some settings, missing _menu attribute'), event_id=event_id)

        return {k: v for k, v in settings.iteritems() if v is not None}
<|endoftext|>


<commit_before>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import sys
import tempfile
import shutil
import subprocess
import glob
import json
import pickle
import logging
from indra.literature import pmc_client, s3_client, get_full_text, \
                             elsevier_client

# Logger
logger = logging.getLogger('runreach')

def upload_reach_json(output_dir, text_sources):
    # At this point, we have a directory full of JSON files
    # Collect all the prefixes into a set, then iterate over the prefixes
    def join_parts(prefix):
        """Join different REACH output JSON files into a single JSON."""
        try:
            with open(prefix + '.uaz.entities.json', 'rt') as f:
                entities = json.load(f)
            with open(prefix + '.uaz.events.json', 'rt') as f:
                events = json.load(f)
            with open(prefix + '.uaz.sentences.json', 'rt') as f:
                sentences = json.load(f)
        except IOError as e:
            logger.error('Failed to open JSON files for %s; REACH error?' %
                          prefix)
            return None
        return {'events': events, 'entities': entities, 'sentences': sentences}

    # Collect prefixes
    json_files = glob.glob(os.path.join(output_dir, '*.json'))
    json_prefixes = set([])
    for json_file in json_files:
        filename = os.path.basename(json_file)
        prefix = filename.split('.')[0]
        json_prefixes.add(prefix)
    # Now iterate over the collected prefixes, combine the JSON, and send to S3
    num_uploaded = 0
    num_failures = 0
    failures = []
    # The prefixes should be PMIDs
    for json_ix, json_prefix in enumerate(json_prefixes):
        prefix_with_path = os.path.join(output_dir, json_prefix)
        full_json = join_parts(prefix_with_path)
        if full_json is None:
            num_failures += 1
            failures.append(json_prefix)
        else:
            # Look up the paper source type
            source_text = text_sources.get(json_prefix)
            logger.info('%s (%d of %d): source %s' %
                      (json_prefix, json_ix + 1, len(json_prefixes), source_text))
            s3_client.put_reach_output(full_json, json_prefix, reach_version,
                                       source_text)
            num_uploaded += 1
    logger.info('Uploaded REACH JSON for %d files to S3 (%d failures)' %
        (num_uploaded, num_failures))
    failures_file = os.path.join(output_dir, 'failures.txt')
    with open(failures_file, 'wt') as f:
        for fail in failures:
            f.write('%s\n' % fail)

if __name__ == '__main__':

    cleanup = False
    verbose = True
    path_to_reach = '/pmc/reach/target/scala-2.11/reach-gordo-1.3.3-SNAPSHOT.jar'
    #path_to_reach = '/Users/johnbachman/Dropbox/1johndata/Knowledge File/Biology/Research/Big Mechanism/reach/target/scala-2.11/reach-gordo-1.3.3-SNAPSHOT.jar'
    reach_version = '1.3.3-38d2c4330'
    force_read = False
    force_fulltext = False

    # Check the arguments
    usage = "Usage: %s pmid_list tmp_dir num_cores start_index end_index " \
            "[force_fulltext]\n" % sys.argv[0]
    usage += "Alternative usage: %s upload_json output_dir content_types_file" % \
              sys.argv[0]
    if len(sys.argv) not in  (4, 6, 7):
        print(usage)
        sys.exit()
    if len(sys.argv) == 4 and sys.argv[1] != 'upload_json':
        print(usage)
        sys.exit()
    if len(sys.argv) == 7 and sys.argv[6] != 'force_fulltext':
        print(usage)
        sys.exit()
    elif len(sys.argv) == 7:
        force_fulltext = True

    # One type of operation: just upload previously read JSON files
    if len(sys.argv) == 4 and sys.argv[1] == 'upload_json':
        output_dir = sys.argv[2]
        text_sources_file = sys.argv[3]
        with open(text_sources_file, 'rb') as f:
            text_sources = pickle.load(f)
        upload_reach_json(output_dir, text_sources)
        sys.exit()

    # =======================
    # Alternatively, run the whole process
    # Get the command line arguments
    (pmid_list_file, tmp_dir, num_cores, start_index, end_index) = sys.argv[1:6]
    start_index = int(start_index)
    end_index = int(end_index)
    num_cores = int(num_cores)

    # Load the list of PMIDs from the given file
    with open(pmid_list_file) as f:
        pmid_list = [line.strip('\n') for line in f.readlines()]
    if end_index > len(pmid_list):
        end_index = len(pmid_list)
    pmids_in_range = pmid_list[start_index:end_index]

    # Create the temp directories for input and output
    base_dir = tempfile.mkdtemp(prefix='read_%s_to_%s_' %
                                (start_index, end_index), dir=tmp_dir)
    input_dir = os.path.join(base_dir, 'input')
    output_dir = os.path.join(base_dir, 'output')
    os.makedirs(input_dir)
    os.makedirs(output_dir)

    pmids_to_read = []

    # If we're re-reading no matter what, we don't have to check for existing
    # REACH output
    if force_read:
        pmids_to_read = pmids_in_range
    # Otherwise, check if we've read the PMIDs already
    else:
        for pmid in pmids_in_range:
            (read_reach_version, read_source_text) = \
                                    s3_client.get_reach_metadata(pmid)
            # Found it, same version
            if read_reach_version is not None and \
               read_reach_version == reach_version:
                logger.info('%s: found same version (%s), skipping' %
                            (pmid, read_reach_version))
            # Found it, different version
            else:
                logger.info('%s: found %s, current %s; will re-read' %
                            (pmid, read_reach_version, reach_version))
                pmids_to_read.append(pmid)

    if not pmids_to_read:
        logger.info('No pmids to read!')
        sys.exit(0)
    else:
        logger.info('Preparing to run REACH on %d PMIDs' % len(pmids_to_read))

    # Now iterate over the pmids to read and download from S3 to the input
    # directory
    num_pmc_oa_xml = 0
    num_pmc_auth_xml = 0
    num_txt = 0
    num_elsevier_xml = 0
    num_abstract = 0
    num_not_found = 0
    num_elsevier_xml_fail = 0
    # Keep a map of the content type we've downloaded for each PMID
    text_sources = {}
    content_not_found = []
    for pmid in pmids_to_read:
        full_pmid = s3_client.check_pmid(pmid)
        # Look for the full text
        (content, content_type) = s3_client.get_upload_content(pmid, force_fulltext_lookup=force_fulltext)
        # If we don't find the XML on S3, look for it using the PMC client
        #if xml:
        #    num_found_s3 += 1
        #else:
        #    logger.info('No content for %s from S3' % pmid)
        #    (content, content_type) = get_full_text(pmid, 'pmid')
        #    if content_type == 'nxml':
        #        logger.info('Found nxml for %s from PMC web service' % pmid)
        #        xml = content
        #        num_found_not_s3 += 1
        #        # Upload the xml to S3 for next time
        #        logger.info('Uploading full text for %s to S3' % pmid)
        #        s3_client.put_full_text(pmid, xml, full_text_type='pmc_oa_xml')
        #    #elif content_type == 'abstract':
        #    #    logger.info('Found abstract for %s' % pmid)
        #    #    s3_client.put_abstract(pmid, content)
        #    else:
        #        logger.info('No full text found for %s' % pmid)
        # Write the contents to a file
        if content_type is None or content is None:
            num_not_found += 1
            content_not_found.append(pmid)
            logger.info('No content found on S3 for %s, skipping' % pmid)
            continue
        elif content_type == 'pmc_oa_xml':
            num_pmc_oa_xml += 1
            text_sources[full_pmid] = 'pmc_oa_xml'
            content_path = os.path.join(input_dir, 'PMID%s.nxml' % pmid)
        elif content_type == 'pmc_auth_xml':
            num_pmc_auth_xml += 1
            text_sources[full_pmid] = 'pmc_auth_xml'
            content_path = os.path.join(input_dir, 'PMID%s.nxml' % pmid)
        elif content_type == 'pmc_oa_txt':
            num_txt += 1
            text_sources[full_pmid] = 'pmc_oa_txt'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'elsevier_xml':
            content = elsevier_client.extract_text(content)
            if content is None:
                logger.info("%s: Couldn't get text from Elsevier XML" % pmid)
                num_elsevier_xml_fail += 1
                continue
            num_elsevier_xml += 1
            text_sources[full_pmid] = 'elsevier_xml'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'txt':
            num_txt += 1
            text_sources[full_pmid] = 'txt'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'abstract':
            num_abstract += 1
            text_sources[full_pmid] = 'abstract'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        else:
            num_not_found += 1
            logger.info('Unhandled content type %s for %s, skipping' %
                        (content_type, pmid))
            continue
        # Write the content to a file with the appropriate extension
        with open(content_path, 'wb') as f:
            # The XML string is Unicode
            enc = content.encode('utf-8')
            f.write(enc)
    logger.info('Saving text sources...')
    text_source_file = os.path.join(base_dir, 'content_types.pkl')
    with open(text_source_file, 'wb') as f:
        pickle.dump(text_sources, f, protocol=2)
    logger.info('Found content PMIDs:')
    logger.info('%d pmc_oa_xml' % num_pmc_oa_xml)
    logger.info('%d pmc_auth_xml' % num_pmc_auth_xml)
    logger.info('%d elsevier_xml' % num_elsevier_xml)
    logger.info('%d elsevier_xml with no full text' % num_elsevier_xml_fail)
    logger.info('%d txt (incl. some Elsevier)' % num_txt)
    logger.info('%d abstract' % num_abstract)
    logger.info('%d no content' % num_not_found)

    # Create the REACH configuration file
    conf_file_text = """
    #
    # Configuration file for reach
    #

    # this is the directory that stores the raw nxml, .csv, and/or .tsv files
    # this directory *must* exist
    papersDir = {input_dir}

    # this is where the output files containing the extracted mentions will be stored
    # if this directory doesn't exist it will be created
    outDir = {output_dir}

    # the output format for mentions: text, fries, indexcard, or assembly-csv (default is 'fries')
    outputType = "fries"

    # whether or not assembly should be run
    withAssembly = false

    # this is where the context files will be stored
    # if this directory doesn't exist it will be created
    contextDir = {output_dir}

    # this is where the brat standoff and text files are dumped
    bratDir = {output_dir}

    # verbose logging
    verbose = true

    # the encoding of input and output files
    encoding = "utf-8"

    # this is a list of sections that we should ignore
    ignoreSections = ["references", "materials", "materials|methods", "methods", "supplementary-material"]

    # context engine config
    contextEngine {{
        type = Policy4
        params = {{
            bound = 3
        }}
    }}

    logging {{
      # defines project-wide logging level
      loglevel = INFO
      logfile = {base_dir}/reach.log
    }}

    # this log file gets overwritten every time ReachCLI is executed
    # so you should copy it if you want to keep it around
    logFile = {base_dir}/log.txt

    # grounding configuration
    grounding: {{
      # List of AdHoc grounding files to insert, in order, into the grounding search sequence.
      # Each element of the list is a map of KB filename and optional meta info (not yet used):
      #   example: {{ kb: "adhoc.tsv", source: "NMZ at CMU" }}
      adHocFiles: [
        {{ kb: "NER-Grounding-Override.tsv.gz", source: "MITRE/NMZ/BG feedback overrides" }}
      ]

      # flag to turn off the influence of species on grounding
      overrideSpecies = true
    }}

    # number of simultaneous threads to use for parallelization
    threadLimit = {num_cores}

    # ReadPapers
    ReadPapers.papersDir = src/test/resources/inputs/nxml/
    ReadPapers.serializedPapers = mentions.ser
""".format(base_dir=base_dir, input_dir=input_dir, output_dir=output_dir,
               num_cores=num_cores)

    # Write the configuration file to the temp directory
    conf_file_path = os.path.join(base_dir, 'indra.conf')
    with open(conf_file_path, 'w') as f:
        f.write(conf_file_text)

    # Run REACH!
    args = ['java', '-jar', path_to_reach, conf_file_path]
    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if verbose:
        for line in iter(p.stdout.readline, b''):
            logger.info(line)
    (p_out, p_err) = p.communicate()
    if p.returncode:
        raise Exception(p_out.decode('utf-8') + '\n' + p_err.decode('utf-8'))

    # Save the list of PMIDs with no content found on S3/literature client
    content_not_found_file = os.path.join(base_dir, 'content_not_found.txt')
    with open(content_not_found_file, 'wt') as f:
        for c in content_not_found:
            f.write('%s\n' % c)

    # Upload!
    upload_reach_json(output_dir, text_sources)

    if cleanup:
        shutil.rmtree(base_dir)

<commit_msg>Update outputType -> outputTypes<commit_after>from __future__ import absolute_import, print_function, unicode_literals
from builtins import dict, str
import os
import sys
import tempfile
import shutil
import subprocess
import glob
import json
import pickle
import logging
from indra.literature import pmc_client, s3_client, get_full_text, \
                             elsevier_client

# Logger
logger = logging.getLogger('runreach')

def upload_reach_json(output_dir, text_sources):
    # At this point, we have a directory full of JSON files
    # Collect all the prefixes into a set, then iterate over the prefixes
    def join_parts(prefix):
        """Join different REACH output JSON files into a single JSON."""
        try:
            with open(prefix + '.uaz.entities.json', 'rt') as f:
                entities = json.load(f)
            with open(prefix + '.uaz.events.json', 'rt') as f:
                events = json.load(f)
            with open(prefix + '.uaz.sentences.json', 'rt') as f:
                sentences = json.load(f)
        except IOError as e:
            logger.error('Failed to open JSON files for %s; REACH error?' %
                          prefix)
            return None
        return {'events': events, 'entities': entities, 'sentences': sentences}

    # Collect prefixes
    json_files = glob.glob(os.path.join(output_dir, '*.json'))
    json_prefixes = set([])
    for json_file in json_files:
        filename = os.path.basename(json_file)
        prefix = filename.split('.')[0]
        json_prefixes.add(prefix)
    # Now iterate over the collected prefixes, combine the JSON, and send to S3
    num_uploaded = 0
    num_failures = 0
    failures = []
    # The prefixes should be PMIDs
    for json_ix, json_prefix in enumerate(json_prefixes):
        prefix_with_path = os.path.join(output_dir, json_prefix)
        full_json = join_parts(prefix_with_path)
        if full_json is None:
            num_failures += 1
            failures.append(json_prefix)
        else:
            # Look up the paper source type
            source_text = text_sources.get(json_prefix)
            logger.info('%s (%d of %d): source %s' %
                      (json_prefix, json_ix + 1, len(json_prefixes), source_text))
            s3_client.put_reach_output(full_json, json_prefix, reach_version,
                                       source_text)
            num_uploaded += 1
    logger.info('Uploaded REACH JSON for %d files to S3 (%d failures)' %
        (num_uploaded, num_failures))
    failures_file = os.path.join(output_dir, 'failures.txt')
    with open(failures_file, 'wt') as f:
        for fail in failures:
            f.write('%s\n' % fail)

if __name__ == '__main__':

    cleanup = False
    verbose = True
    path_to_reach = '/pmc/reach/target/scala-2.11/reach-gordo-1.3.3-SNAPSHOT.jar'
    #path_to_reach = '/Users/johnbachman/Dropbox/1johndata/Knowledge File/Biology/Research/Big Mechanism/reach/target/scala-2.11/reach-gordo-1.3.3-SNAPSHOT.jar'
    reach_version = '1.3.3-38d2c4330'
    force_read = False
    force_fulltext = False

    # Check the arguments
    usage = "Usage: %s pmid_list tmp_dir num_cores start_index end_index " \
            "[force_fulltext]\n" % sys.argv[0]
    usage += "Alternative usage: %s upload_json output_dir content_types_file" % \
              sys.argv[0]
    if len(sys.argv) not in  (4, 6, 7):
        print(usage)
        sys.exit()
    if len(sys.argv) == 4 and sys.argv[1] != 'upload_json':
        print(usage)
        sys.exit()
    if len(sys.argv) == 7 and sys.argv[6] != 'force_fulltext':
        print(usage)
        sys.exit()
    elif len(sys.argv) == 7:
        force_fulltext = True

    # One type of operation: just upload previously read JSON files
    if len(sys.argv) == 4 and sys.argv[1] == 'upload_json':
        output_dir = sys.argv[2]
        text_sources_file = sys.argv[3]
        with open(text_sources_file, 'rb') as f:
            text_sources = pickle.load(f)
        upload_reach_json(output_dir, text_sources)
        sys.exit()

    # =======================
    # Alternatively, run the whole process
    # Get the command line arguments
    (pmid_list_file, tmp_dir, num_cores, start_index, end_index) = sys.argv[1:6]
    start_index = int(start_index)
    end_index = int(end_index)
    num_cores = int(num_cores)

    # Load the list of PMIDs from the given file
    with open(pmid_list_file) as f:
        pmid_list = [line.strip('\n') for line in f.readlines()]
    if end_index > len(pmid_list):
        end_index = len(pmid_list)
    pmids_in_range = pmid_list[start_index:end_index]

    # Create the temp directories for input and output
    base_dir = tempfile.mkdtemp(prefix='read_%s_to_%s_' %
                                (start_index, end_index), dir=tmp_dir)
    input_dir = os.path.join(base_dir, 'input')
    output_dir = os.path.join(base_dir, 'output')
    os.makedirs(input_dir)
    os.makedirs(output_dir)

    pmids_to_read = []

    # If we're re-reading no matter what, we don't have to check for existing
    # REACH output
    if force_read:
        pmids_to_read = pmids_in_range
    # Otherwise, check if we've read the PMIDs already
    else:
        for pmid in pmids_in_range:
            (read_reach_version, read_source_text) = \
                                    s3_client.get_reach_metadata(pmid)
            # Found it, same version
            if read_reach_version is not None and \
               read_reach_version == reach_version:
                logger.info('%s: found same version (%s), skipping' %
                            (pmid, read_reach_version))
            # Found it, different version
            else:
                logger.info('%s: found %s, current %s; will re-read' %
                            (pmid, read_reach_version, reach_version))
                pmids_to_read.append(pmid)

    if not pmids_to_read:
        logger.info('No pmids to read!')
        sys.exit(0)
    else:
        logger.info('Preparing to run REACH on %d PMIDs' % len(pmids_to_read))

    # Now iterate over the pmids to read and download from S3 to the input
    # directory
    num_pmc_oa_xml = 0
    num_pmc_auth_xml = 0
    num_txt = 0
    num_elsevier_xml = 0
    num_abstract = 0
    num_not_found = 0
    num_elsevier_xml_fail = 0
    # Keep a map of the content type we've downloaded for each PMID
    text_sources = {}
    content_not_found = []
    for pmid in pmids_to_read:
        full_pmid = s3_client.check_pmid(pmid)
        # Look for the full text
        (content, content_type) = s3_client.get_upload_content(pmid, force_fulltext_lookup=force_fulltext)
        # If we don't find the XML on S3, look for it using the PMC client
        #if xml:
        #    num_found_s3 += 1
        #else:
        #    logger.info('No content for %s from S3' % pmid)
        #    (content, content_type) = get_full_text(pmid, 'pmid')
        #    if content_type == 'nxml':
        #        logger.info('Found nxml for %s from PMC web service' % pmid)
        #        xml = content
        #        num_found_not_s3 += 1
        #        # Upload the xml to S3 for next time
        #        logger.info('Uploading full text for %s to S3' % pmid)
        #        s3_client.put_full_text(pmid, xml, full_text_type='pmc_oa_xml')
        #    #elif content_type == 'abstract':
        #    #    logger.info('Found abstract for %s' % pmid)
        #    #    s3_client.put_abstract(pmid, content)
        #    else:
        #        logger.info('No full text found for %s' % pmid)
        # Write the contents to a file
        if content_type is None or content is None:
            num_not_found += 1
            content_not_found.append(pmid)
            logger.info('No content found on S3 for %s, skipping' % pmid)
            continue
        elif content_type == 'pmc_oa_xml':
            num_pmc_oa_xml += 1
            text_sources[full_pmid] = 'pmc_oa_xml'
            content_path = os.path.join(input_dir, 'PMID%s.nxml' % pmid)
        elif content_type == 'pmc_auth_xml':
            num_pmc_auth_xml += 1
            text_sources[full_pmid] = 'pmc_auth_xml'
            content_path = os.path.join(input_dir, 'PMID%s.nxml' % pmid)
        elif content_type == 'pmc_oa_txt':
            num_txt += 1
            text_sources[full_pmid] = 'pmc_oa_txt'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'elsevier_xml':
            content = elsevier_client.extract_text(content)
            if content is None:
                logger.info("%s: Couldn't get text from Elsevier XML" % pmid)
                num_elsevier_xml_fail += 1
                continue
            num_elsevier_xml += 1
            text_sources[full_pmid] = 'elsevier_xml'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'txt':
            num_txt += 1
            text_sources[full_pmid] = 'txt'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        elif content_type == 'abstract':
            num_abstract += 1
            text_sources[full_pmid] = 'abstract'
            content_path = os.path.join(input_dir, 'PMID%s.txt' % pmid)
        else:
            num_not_found += 1
            logger.info('Unhandled content type %s for %s, skipping' %
                        (content_type, pmid))
            continue
        # Write the content to a file with the appropriate extension
        with open(content_path, 'wb') as f:
            # The XML string is Unicode
            enc = content.encode('utf-8')
            f.write(enc)
    logger.info('Saving text sources...')
    text_source_file = os.path.join(base_dir, 'content_types.pkl')
    with open(text_source_file, 'wb') as f:
        pickle.dump(text_sources, f, protocol=2)
    logger.info('Found content PMIDs:')
    logger.info('%d pmc_oa_xml' % num_pmc_oa_xml)
    logger.info('%d pmc_auth_xml' % num_pmc_auth_xml)
    logger.info('%d elsevier_xml' % num_elsevier_xml)
    logger.info('%d elsevier_xml with no full text' % num_elsevier_xml_fail)
    logger.info('%d txt (incl. some Elsevier)' % num_txt)
    logger.info('%d abstract' % num_abstract)
    logger.info('%d no content' % num_not_found)

    # Create the REACH configuration file
    conf_file_text = """
    #
    # Configuration file for reach
    #

    # this is the directory that stores the raw nxml, .csv, and/or .tsv files
    # this directory *must* exist
    papersDir = {input_dir}

    # this is where the output files containing the extracted mentions will be stored
    # if this directory doesn't exist it will be created
    outDir = {output_dir}

    # the output format for mentions: text, fries, indexcard, or assembly-csv (default is 'fries')
    outputTypes = ["fries"]

    # whether or not assembly should be run
    withAssembly = false

    # this is where the context files will be stored
    # if this directory doesn't exist it will be created
    contextDir = {output_dir}

    # this is where the brat standoff and text files are dumped
    bratDir = {output_dir}

    # verbose logging
    verbose = true

    # the encoding of input and output files
    encoding = "utf-8"

    # this is a list of sections that we should ignore
    ignoreSections = ["references", "materials", "materials|methods", "methods", "supplementary-material"]

    # context engine config
    contextEngine {{
        type = Policy4
        params = {{
            bound = 3
        }}
    }}

    logging {{
      # defines project-wide logging level
      loglevel = INFO
      logfile = {base_dir}/reach.log
    }}

    # this log file gets overwritten every time ReachCLI is executed
    # so you should copy it if you want to keep it around
    logFile = {base_dir}/log.txt

    # grounding configuration
    grounding: {{
      # List of AdHoc grounding files to insert, in order, into the grounding search sequence.
      # Each element of the list is a map of KB filename and optional meta info (not yet used):
      #   example: {{ kb: "adhoc.tsv", source: "NMZ at CMU" }}
      adHocFiles: [
        {{ kb: "NER-Grounding-Override.tsv.gz", source: "MITRE/NMZ/BG feedback overrides" }}
      ]

      # flag to turn off the influence of species on grounding
      overrideSpecies = true
    }}

    # number of simultaneous threads to use for parallelization
    threadLimit = {num_cores}

    # ReadPapers
    ReadPapers.papersDir = src/test/resources/inputs/nxml/
    ReadPapers.serializedPapers = mentions.ser
""".format(base_dir=base_dir, input_dir=input_dir, output_dir=output_dir,
               num_cores=num_cores)

    # Write the configuration file to the temp directory
    conf_file_path = os.path.join(base_dir, 'indra.conf')
    with open(conf_file_path, 'w') as f:
        f.write(conf_file_text)

    # Run REACH!
    args = ['java', '-jar', path_to_reach, conf_file_path]
    p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if verbose:
        for line in iter(p.stdout.readline, b''):
            logger.info(line)
    (p_out, p_err) = p.communicate()
    if p.returncode:
        raise Exception(p_out.decode('utf-8') + '\n' + p_err.decode('utf-8'))

    # Save the list of PMIDs with no content found on S3/literature client
    content_not_found_file = os.path.join(base_dir, 'content_not_found.txt')
    with open(content_not_found_file, 'wt') as f:
        for c in content_not_found:
            f.write('%s\n' % c)

    # Upload!
    upload_reach_json(output_dir, text_sources)

    if cleanup:
        shutil.rmtree(base_dir)

<|endoftext|>


<commit_before># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import logging
import os
import shutil
import subprocess
import tempfile
import traceback
import time

from inferlib import config, issues, utils, bucklib
from . import util

import re

MODULE_NAME = __name__
MODULE_DESCRIPTION = '''Run analysis of code built with a command like:
buck [options] [target]

Analysis examples:
infer -- buck build HelloWorld'''
LANG = ['clang', 'java']

KEEP_GOING_OPTION = "--keep-going"

def gen_instance(*args):
    return BuckAnalyzer(*args)

def string_in_quotes(value):
    return value.strip('\'')

def create_argparser(group_name=MODULE_NAME):
    """This defines the set of arguments that get added by this module to the
    set of global args defined in the infer top-level module
    Do not use this function directly, it should be invoked by the infer
    top-level module"""
    parser = argparse.ArgumentParser(add_help=False)
    group = parser.add_argument_group(
        '{grp} module'.format(grp=MODULE_NAME),
        description=MODULE_DESCRIPTION,
    )
    group.add_argument('--use-flavors', action='store_true',
                       help='Run Infer analysis through the use of flavors. '
                            'Currently this is supported only for the cxx_* '
                            'targets of Buck - e.g. cxx_library, cxx_binary - '
                            'and not for Java. Note: this flag should be used '
                            'in combination with passing the #infer flavor '
                            'to the Buck target.')
    group.add_argument('--xcode-developer-dir',
                       help='Specify the path to Xcode developer directory '
                            '(requires --use-flavors to work)')
    group.add_argument('--blacklist-regex',
                       help='Specify the regex for files to skip during '
                            'the analysis (requires --use-flavors to work)')
    group.add_argument('--Xbuck', action='append', default=[],
                       type=string_in_quotes,
                       help='Pass values as command-line arguments to '
                            'invocations of `buck build`.'
                            'NOTE: value should be wrapped in single quotes')

    return parser

class BuckAnalyzer:
    def __init__(self, args, cmd):
        self.args = args
        self.cmd = cmd
        self.keep_going = KEEP_GOING_OPTION in self.args.Xbuck
        util.log_java_version()
        logging.info(util.run_cmd_ignore_fail(['buck', '--version']))

    def capture(self):
        try:
            if self.args.use_flavors:
                return self.capture_with_flavors()
            else:
                return self.capture_without_flavors()
        except subprocess.CalledProcessError as exc:
            if self.args.debug:
                traceback.print_exc()
            return exc.returncode

    def create_cxx_buck_configuration_args(self):
        # return a string that can be passed in input to buck
        # and configures the paths to infer/clang/plugin/xcode
        facebook_clang_plugins_root = config.FCP_DIRECTORY
        clang_path = os.path.join(
            facebook_clang_plugins_root,
            'clang',
            'install',
            'bin',
            'clang',
        )
        plugin_path = os.path.join(
            facebook_clang_plugins_root,
            'libtooling',
            'build',
            'FacebookClangPlugin.dylib',
        )
        args = [
            '--config',
            'client.id=infer.clang',
            '--config',
            '*//infer.infer_bin={bin}'
            .format(bin=config.BIN_DIRECTORY),
            '--config',
            '*//infer.clang_compiler={clang}'.format(clang=clang_path),
            '--config',
            '*//infer.clang_plugin={plugin}'.format(plugin=plugin_path),
            '--config',
            '*//cxx.pch_enabled=false',
            '--config',  # Infer doesn't support C++ modules yet (T35656509)
            '*//cxx.modules_default=false',
            '--config',
            '*//cxx.modules=False',
        ] + self.args.Xbuck

        if self.args.xcode_developer_dir is not None:
            args.append('--config')
            args.append('apple.xcode_developer_dir={devdir}'.format(
                devdir=self.args.xcode_developer_dir))
        if self.args.blacklist_regex:
            args.append('--config')
            args.append('*//infer.blacklist_regex={regex}'.format(
                regex=self.args.blacklist_regex))
        return args

    def _get_analysis_result_paths(self):
        # TODO(8610738): Make targets extraction smarter
        buck_results_cmd = [
            self.cmd[0],
            'targets',
            '--show-output'
        ] + self.cmd[2:] + self.create_cxx_buck_configuration_args()
        buck_results_cmd = \
            [x for x in buck_results_cmd if x != KEEP_GOING_OPTION]
        proc = subprocess.Popen(buck_results_cmd, stdout=subprocess.PIPE)
        (buck_output, _) = proc.communicate()
        if proc.returncode != 0:
            return None
        # remove target name prefixes from each line and split them into a list
        out = [x.split(None, 1)[1] for x in buck_output.strip().split('\n')]
        return [os.path.dirname(x)
                if os.path.isfile(x) else x
                for x in out if os.path.exists(x)]

    @staticmethod
    def _merge_infer_dep_files(root_paths, merged_out_path):
        potential_dep_files = [os.path.join(p, config.INFER_BUCK_DEPS_FILENAME)
                               for p in root_paths]
        dep_files = filter(os.path.exists, potential_dep_files)
        utils.merge_and_dedup_files_into_path(dep_files, merged_out_path)

    @staticmethod
    def _merge_infer_report_files(root_paths, merged_out_path):
        potential_report_files = [os.path.join(p, config.JSON_REPORT_FILENAME)
                                  for p in root_paths]
        report_files = filter(os.path.exists, potential_report_files)
        all_results = issues.merge_reports_from_paths(report_files)
        utils.dump_json_to_path(all_results, merged_out_path)

    @staticmethod
    def _find_deps_and_merge(merged_out_path):
        """This function is used to compute the infer-deps.txt file that
        contains the location of the infer-out folders with the captured
        files created by buck. This is needed when keep-going is passed
        to buck and there are compilation failures, because in that case
        buck doesn't create this file."""
        infer_out_folders = []
        start_time = time.time()
        print('finding captured files in buck-out...')
        for root, dirs, files in os.walk(config.BUCK_OUT_GEN):
            regex = re.compile('.*infer-out.*')
            folders = \
                [os.path.join(root, d) for d in dirs if re.match(regex, d)]
            for d in folders:
                if d not in infer_out_folders:
                    infer_out_folders.append(d)
        with open(merged_out_path, 'w') as fmerged_out_path:
            for dir in infer_out_folders:
                fmerged_out_path.write('\t' + '\t' + dir + '\n')
        elapsed_time = time.time() - start_time
        print('time elapsed in finding captured files in buck-out: % 6.2fs'
              % elapsed_time)

    def _move_buck_out(self):
        """ If keep-going is passed, we may need to compute the infer-deps
        file with the paths to the captured files. To make sure that
        this is done in a consistent way, we need to start the analysis
        with an empty buck-out folder."""
        if not os.path.exists(config.BUCK_OUT_TRASH):
            os.makedirs(config.BUCK_OUT_TRASH)
        tmp = tempfile.mkdtemp(
              dir=config.BUCK_OUT_TRASH,
              prefix=config.BUCK_OUT)
        print('moving files in ' + config.BUCK_OUT + ' to ' + tmp)
        for filename in os.listdir(config.BUCK_OUT):
            if filename != config.TRASH:
                shutil.move(os.path.join(config.BUCK_OUT, filename), tmp)

    def _run_buck_with_flavors(self):
        # TODO: Use buck to identify the project's root folder
        if not os.path.isfile('.buckconfig'):
            print('Please run this command from the folder where .buckconfig '
                  'is located')
            return os.EX_USAGE
        env_vars = utils.read_env()
        infer_args = env_vars['INFER_ARGS']
        if infer_args != '':
            infer_args += '^'  # '^' must be CommandLineOption.env_var_sep
        infer_args += '--fcp-syntax-only'
        env_vars['INFER_ARGS'] = infer_args
        env = utils.encode_env(env_vars)
        command = self.cmd
        command += ['-j', str(self.args.multicore)]
        if self.args.load_average is not None:
            command += ['-L', str(self.args.load_average)]
        command += self.create_cxx_buck_configuration_args()
        try:
            subprocess.check_call(command, env=env)
            return os.EX_OK
        except subprocess.CalledProcessError as e:
            if self.keep_going:
                print('Buck failed, but continuing the analysis '
                      'because --keep-going was passed')
                return -1
            else:
                raise e

    def capture_with_flavors(self):
        if self.keep_going and not self.args.continue_capture:
            self._move_buck_out()
        ret = self._run_buck_with_flavors()
        if not ret == os.EX_OK and not self.keep_going:
            return ret
        result_paths = self._get_analysis_result_paths()
        if result_paths is None:
            # huho, the Buck command to extract results paths failed
            return os.EX_SOFTWARE
        merged_reports_path = os.path.join(
            self.args.infer_out, config.JSON_REPORT_FILENAME)
        merged_deps_path = os.path.join(
            self.args.infer_out, config.INFER_BUCK_DEPS_FILENAME)
        self._merge_infer_report_files(result_paths, merged_reports_path)
        if not ret == os.EX_OK and self.keep_going:
            self._find_deps_and_merge(merged_deps_path)
        else:
            self._merge_infer_dep_files(result_paths, merged_deps_path)
        infer_out = self.args.infer_out
        json_report = os.path.join(infer_out, config.JSON_REPORT_FILENAME)
        bugs_out = os.path.join(infer_out, config.BUGS_FILENAME)
        issues.print_and_save_errors(infer_out, self.args.project_root,
                                     json_report, bugs_out, self.args.pmd_xml,
                                     console_out=not self.args.quiet)
        return os.EX_OK

    def capture_without_flavors(self):
        # Java is a special case, and we run the analysis from here
        buck_wrapper = bucklib.Wrapper(self.args, self.cmd)
        return buck_wrapper.run()
<commit_msg>buck integration, no lonfer bail when .buckconfig is missing<commit_after># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import logging
import os
import shutil
import subprocess
import tempfile
import traceback
import time

from inferlib import config, issues, utils, bucklib
from . import util

import re

MODULE_NAME = __name__
MODULE_DESCRIPTION = '''Run analysis of code built with a command like:
buck [options] [target]

Analysis examples:
infer -- buck build HelloWorld'''
LANG = ['clang', 'java']

KEEP_GOING_OPTION = "--keep-going"

def gen_instance(*args):
    return BuckAnalyzer(*args)

def string_in_quotes(value):
    return value.strip('\'')

def create_argparser(group_name=MODULE_NAME):
    """This defines the set of arguments that get added by this module to the
    set of global args defined in the infer top-level module
    Do not use this function directly, it should be invoked by the infer
    top-level module"""
    parser = argparse.ArgumentParser(add_help=False)
    group = parser.add_argument_group(
        '{grp} module'.format(grp=MODULE_NAME),
        description=MODULE_DESCRIPTION,
    )
    group.add_argument('--use-flavors', action='store_true',
                       help='Run Infer analysis through the use of flavors. '
                            'Currently this is supported only for the cxx_* '
                            'targets of Buck - e.g. cxx_library, cxx_binary - '
                            'and not for Java. Note: this flag should be used '
                            'in combination with passing the #infer flavor '
                            'to the Buck target.')
    group.add_argument('--xcode-developer-dir',
                       help='Specify the path to Xcode developer directory '
                            '(requires --use-flavors to work)')
    group.add_argument('--blacklist-regex',
                       help='Specify the regex for files to skip during '
                            'the analysis (requires --use-flavors to work)')
    group.add_argument('--Xbuck', action='append', default=[],
                       type=string_in_quotes,
                       help='Pass values as command-line arguments to '
                            'invocations of `buck build`.'
                            'NOTE: value should be wrapped in single quotes')

    return parser

class BuckAnalyzer:
    def __init__(self, args, cmd):
        self.args = args
        self.cmd = cmd
        self.keep_going = KEEP_GOING_OPTION in self.args.Xbuck
        util.log_java_version()
        logging.info(util.run_cmd_ignore_fail(['buck', '--version']))

    def capture(self):
        try:
            if self.args.use_flavors:
                return self.capture_with_flavors()
            else:
                return self.capture_without_flavors()
        except subprocess.CalledProcessError as exc:
            if self.args.debug:
                traceback.print_exc()
            return exc.returncode

    def create_cxx_buck_configuration_args(self):
        # return a string that can be passed in input to buck
        # and configures the paths to infer/clang/plugin/xcode
        facebook_clang_plugins_root = config.FCP_DIRECTORY
        clang_path = os.path.join(
            facebook_clang_plugins_root,
            'clang',
            'install',
            'bin',
            'clang',
        )
        plugin_path = os.path.join(
            facebook_clang_plugins_root,
            'libtooling',
            'build',
            'FacebookClangPlugin.dylib',
        )
        args = [
            '--config',
            'client.id=infer.clang',
            '--config',
            '*//infer.infer_bin={bin}'
            .format(bin=config.BIN_DIRECTORY),
            '--config',
            '*//infer.clang_compiler={clang}'.format(clang=clang_path),
            '--config',
            '*//infer.clang_plugin={plugin}'.format(plugin=plugin_path),
            '--config',
            '*//cxx.pch_enabled=false',
            '--config',  # Infer doesn't support C++ modules yet (T35656509)
            '*//cxx.modules_default=false',
            '--config',
            '*//cxx.modules=False',
        ] + self.args.Xbuck

        if self.args.xcode_developer_dir is not None:
            args.append('--config')
            args.append('apple.xcode_developer_dir={devdir}'.format(
                devdir=self.args.xcode_developer_dir))
        if self.args.blacklist_regex:
            args.append('--config')
            args.append('*//infer.blacklist_regex={regex}'.format(
                regex=self.args.blacklist_regex))
        return args

    def _get_analysis_result_paths(self):
        # TODO(8610738): Make targets extraction smarter
        buck_results_cmd = [
            self.cmd[0],
            'targets',
            '--show-output'
        ] + self.cmd[2:] + self.create_cxx_buck_configuration_args()
        buck_results_cmd = \
            [x for x in buck_results_cmd if x != KEEP_GOING_OPTION]
        proc = subprocess.Popen(buck_results_cmd, stdout=subprocess.PIPE)
        (buck_output, _) = proc.communicate()
        if proc.returncode != 0:
            return None
        # remove target name prefixes from each line and split them into a list
        out = [x.split(None, 1)[1] for x in buck_output.strip().split('\n')]
        return [os.path.dirname(x)
                if os.path.isfile(x) else x
                for x in out if os.path.exists(x)]

    @staticmethod
    def _merge_infer_dep_files(root_paths, merged_out_path):
        potential_dep_files = [os.path.join(p, config.INFER_BUCK_DEPS_FILENAME)
                               for p in root_paths]
        dep_files = filter(os.path.exists, potential_dep_files)
        utils.merge_and_dedup_files_into_path(dep_files, merged_out_path)

    @staticmethod
    def _merge_infer_report_files(root_paths, merged_out_path):
        potential_report_files = [os.path.join(p, config.JSON_REPORT_FILENAME)
                                  for p in root_paths]
        report_files = filter(os.path.exists, potential_report_files)
        all_results = issues.merge_reports_from_paths(report_files)
        utils.dump_json_to_path(all_results, merged_out_path)

    @staticmethod
    def _find_deps_and_merge(merged_out_path):
        """This function is used to compute the infer-deps.txt file that
        contains the location of the infer-out folders with the captured
        files created by buck. This is needed when keep-going is passed
        to buck and there are compilation failures, because in that case
        buck doesn't create this file."""
        infer_out_folders = []
        start_time = time.time()
        print('finding captured files in buck-out...')
        for root, dirs, files in os.walk(config.BUCK_OUT_GEN):
            regex = re.compile('.*infer-out.*')
            folders = \
                [os.path.join(root, d) for d in dirs if re.match(regex, d)]
            for d in folders:
                if d not in infer_out_folders:
                    infer_out_folders.append(d)
        with open(merged_out_path, 'w') as fmerged_out_path:
            for dir in infer_out_folders:
                fmerged_out_path.write('\t' + '\t' + dir + '\n')
        elapsed_time = time.time() - start_time
        print('time elapsed in finding captured files in buck-out: % 6.2fs'
              % elapsed_time)

    def _move_buck_out(self):
        """ If keep-going is passed, we may need to compute the infer-deps
        file with the paths to the captured files. To make sure that
        this is done in a consistent way, we need to start the analysis
        with an empty buck-out folder."""
        if not os.path.exists(config.BUCK_OUT_TRASH):
            os.makedirs(config.BUCK_OUT_TRASH)
        tmp = tempfile.mkdtemp(
              dir=config.BUCK_OUT_TRASH,
              prefix=config.BUCK_OUT)
        print('moving files in ' + config.BUCK_OUT + ' to ' + tmp)
        for filename in os.listdir(config.BUCK_OUT):
            if filename != config.TRASH:
                shutil.move(os.path.join(config.BUCK_OUT, filename), tmp)

    def _run_buck_with_flavors(self):
        env_vars = utils.read_env()
        infer_args = env_vars['INFER_ARGS']
        if infer_args != '':
            infer_args += '^'  # '^' must be CommandLineOption.env_var_sep
        infer_args += '--fcp-syntax-only'
        env_vars['INFER_ARGS'] = infer_args
        env = utils.encode_env(env_vars)
        command = self.cmd
        command += ['-j', str(self.args.multicore)]
        if self.args.load_average is not None:
            command += ['-L', str(self.args.load_average)]
        command += self.create_cxx_buck_configuration_args()
        try:
            subprocess.check_call(command, env=env)
            return os.EX_OK
        except subprocess.CalledProcessError as e:
            if self.keep_going:
                print('Buck failed, but continuing the analysis '
                      'because --keep-going was passed')
                return -1
            else:
                raise e

    def capture_with_flavors(self):
        if self.keep_going and not self.args.continue_capture:
            self._move_buck_out()
        ret = self._run_buck_with_flavors()
        if not ret == os.EX_OK and not self.keep_going:
            return ret
        result_paths = self._get_analysis_result_paths()
        if result_paths is None:
            # huho, the Buck command to extract results paths failed
            return os.EX_SOFTWARE
        merged_reports_path = os.path.join(
            self.args.infer_out, config.JSON_REPORT_FILENAME)
        merged_deps_path = os.path.join(
            self.args.infer_out, config.INFER_BUCK_DEPS_FILENAME)
        self._merge_infer_report_files(result_paths, merged_reports_path)
        if not ret == os.EX_OK and self.keep_going:
            self._find_deps_and_merge(merged_deps_path)
        else:
            self._merge_infer_dep_files(result_paths, merged_deps_path)
        infer_out = self.args.infer_out
        json_report = os.path.join(infer_out, config.JSON_REPORT_FILENAME)
        bugs_out = os.path.join(infer_out, config.BUGS_FILENAME)
        issues.print_and_save_errors(infer_out, self.args.project_root,
                                     json_report, bugs_out, self.args.pmd_xml,
                                     console_out=not self.args.quiet)
        return os.EX_OK

    def capture_without_flavors(self):
        # Java is a special case, and we run the analysis from here
        buck_wrapper = bucklib.Wrapper(self.args, self.cmd)
        return buck_wrapper.run()
<|endoftext|>