<commit_before># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2016, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import os
import io
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from lxml import etree

from .test_models import BaseModelTestCase

here = os.path.abspath(os.path.dirname(__file__))
TEST_DATA_DIR = os.path.join(here, 'data')

class ReconstituteTestCase(unittest.TestCase):
    maxDiff = None

    def test_xhtml(self):
        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.xhtml')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)
        self.check_desserts(desserts)

    def test_html(self):
        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.html')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)
        self.check_desserts(desserts)

    def check_desserts(self, desserts):
        """Assertions for the desserts model"""
        from ..models import model_to_tree

        self.assertEqual('Desserts', desserts.metadata['title'])

        self.assertEqual({
            'id': 'book',
            'title': 'Desserts',
            'contents': [
                {
                    'id': 'subcol',
                    'title': 'Fruity',
                    'contents': [
                        {
                            'id': 'apple',
                            'title': 'Apple',
                            },
                        {
                            'id': 'lemon',
                            'title': '1.1 | '
                                     'レモン',
                            },
                        {
                            'id': 'subcol',
                            'title': 'Chapter 2 '
                                     'citrus',
                            'contents': [
                                {
                                    'id': 'lemon',
                                    'title': 'Lemon',
                                    },
                                ],
                            },
                        ],
                    },
                {
                    'id': 'chocolate',
                    'title': u'チョコレート',
                    },
                {
                    'id': 'extra',
                    'title': 'Extra Stuff',
                    },
                ],
            }, model_to_tree(desserts))

        base_metadata = {
            u'publishers': [],
            u'created': None,  # '2016/03/04 17:05:20 -0500',
            u'revised': None,  # '2013/03/05 09:35:24 -0500',
            u'authors': [
                {u'type': u'cnx-id',
                 u'name': u'Good Food',
                 u'id': u'yum'}],
            u'editors': [],
            u'copyright_holders': [],
            u'illustrators': [],
            u'subjects': [u'Humanities'],
            u'translators': [],
            u'keywords': [u'Food', u'デザート', u'Pudding'],
            u'title': u'チョコレート',
            u'license_text': u'CC-By 4.0',
            u'license_url': u'http://creativecommons.org/licenses/by/4.0/',
            # 'version': 'draft',
            u'language': None,
            u'print_style': None,
            u'cnx-archive-uri': None,
            u'derived_from_title': None,
            u'derived_from_uri': None,
            }

        fruity = desserts[0]
        self.assertEqual('TranslucentBinder', fruity.__class__.__name__)
        self.assertEqual('Fruity', fruity.metadata['title'])

        apple = fruity[0]
        self.assertEqual('Document', apple.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Apple'
        apple_metadata = apple.metadata.copy()
        summary = etree.fromstring(apple_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, apple_metadata)

        lemon = fruity[1]
        self.assertEqual('Document', lemon.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Lemon'
        lemon_metadata = lemon.metadata.copy()
        summary = etree.fromstring(lemon_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, lemon_metadata)

        citrus = fruity[2]
        self.assertEqual('TranslucentBinder', citrus.__class__.__name__)
        self.assertEqual(citrus.metadata['title'], 'Citrus')

        self.assertEqual(lemon.metadata, citrus[0].metadata)

        chocolate = desserts[1]
        self.assertEqual('Document', chocolate.__class__.__name__)
        chocolate_metadata = chocolate.metadata.copy()
        summary = etree.fromstring(chocolate_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = u'チョコレート'
        self.assertEqual(metadata, chocolate_metadata)

        extra = desserts[2]
        self.assertEqual('CompositeDocument', extra.__class__.__name__)
        extra_metadata = extra.metadata.copy()
        summary = etree.fromstring(extra_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = 'Extra Stuff'
        self.assertEqual(metadata, extra_metadata)

class CollateTestCase(BaseModelTestCase):

    @property
    def target(self):
        from cnxepub.collation import collate
        return collate

    def test(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"
document one
",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"
document two
",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b" ")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        def mock_easybake(ruleset, in_html, out_html):
            from lxml import etree
            html = etree.parse(in_html)
            # Add in a composite-page with title "Composite One" here.
            body = html.getroot().xpath(
                '//xhtml:body',
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            comp_elm = etree.SubElement(body, 'div')
            comp_elm.attrib['data-type'] = 'composite-page'
            comp_elm.append(etree.fromstring("""

Composite One

                By:
                Edited by:
                Illustrated by:
                Translated by:

                Published By:

                Licensed:

"""))
            etree.SubElement(comp_elm, 'p').text = "composite document"
            # Add the composite-page to the table-of-contents.
            toc = html.getroot().xpath(
                "//xhtml:*[@id='toc']/xhtml:ol",
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            etree.SubElement(toc, 'li').append(etree.fromstring('Composite One'))
            out_html.write(etree.tostring(html))

        with mock.patch('cnxepub.collation.easybake') as easybake:
            easybake.side_effect = mock_easybake
            collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].id, None)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Composite One')

    def test_without_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        result = self.target(binder)
        self.assertIs(binder, result)

    def test_with_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"document one",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"document two",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b"""\
div[data-type='page'] > div[data-type='metadata'] {
  copy-to: eob-all
}
div[data-type='page'] span {
  copy-to: eob-all
}
body::after {
  content: pending(eob-all);
  class: end-of-book;
  data-type: composite-page;
  container: div;
}

/* copied from cte books/rulesets/common/toc.less */
body > div[data-type="page"],
body > div[data-type="composite-page"] {
  pass: 20;
  string-set: page-id attr(id);
}
body > div[data-type="page"] > div[data-type="metadata"] > \
    h1[data-type='document-title'],
body > div[data-type="composite-page"] > div[data-type="metadata"] > \
    h1[data-type='document-title'] {
  pass: 20;
  copy-to: page-title;
}
body > div[data-type="page"]::after,
body > div[data-type="composite-page"]::after {
  pass: 20;
  content: pending(page-title);
  attr-href: "#" string(page-id);
  container: a;
  move-to: page-link;
}
body > div[data-type="page"]::after,
body > div[data-type="composite-page"]::after {
  pass: 20;
  content: pending(page-link);
  move-to: eob-toc;
  container: li;
}
nav#toc {
  pass: 30;
  content: '';
}
nav#toc::after {
  pass: 30;
  content: pending(eob-toc);
  container: ol;
}
""")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Document One')
<commit_msg>convert ruleset to pass-in-selector syntax<commit_after># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2016, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import os
import io
import unittest
try:
    from unittest import mock
except ImportError:
    import mock

from lxml import etree

from .test_models import BaseModelTestCase

here = os.path.abspath(os.path.dirname(__file__))
TEST_DATA_DIR = os.path.join(here, 'data')

class ReconstituteTestCase(unittest.TestCase):
    maxDiff = None

    def test_xhtml(self):
        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.xhtml')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)
        self.check_desserts(desserts)

    def test_html(self):
        page_path = os.path.join(TEST_DATA_DIR, 'desserts-single-page.html')
        with open(page_path) as html:
            from cnxepub.collation import reconstitute
            desserts = reconstitute(html)
        self.check_desserts(desserts)

    def check_desserts(self, desserts):
        """Assertions for the desserts model"""
        from ..models import model_to_tree

        self.assertEqual('Desserts', desserts.metadata['title'])

        self.assertEqual({
            'id': 'book',
            'title': 'Desserts',
            'contents': [
                {
                    'id': 'subcol',
                    'title': 'Fruity',
                    'contents': [
                        {
                            'id': 'apple',
                            'title': 'Apple',
                            },
                        {
                            'id': 'lemon',
                            'title': '1.1 | '
                                     'レモン',
                            },
                        {
                            'id': 'subcol',
                            'title': 'Chapter 2 '
                                     'citrus',
                            'contents': [
                                {
                                    'id': 'lemon',
                                    'title': 'Lemon',
                                    },
                                ],
                            },
                        ],
                    },
                {
                    'id': 'chocolate',
                    'title': u'チョコレート',
                    },
                {
                    'id': 'extra',
                    'title': 'Extra Stuff',
                    },
                ],
            }, model_to_tree(desserts))

        base_metadata = {
            u'publishers': [],
            u'created': None,  # '2016/03/04 17:05:20 -0500',
            u'revised': None,  # '2013/03/05 09:35:24 -0500',
            u'authors': [
                {u'type': u'cnx-id',
                 u'name': u'Good Food',
                 u'id': u'yum'}],
            u'editors': [],
            u'copyright_holders': [],
            u'illustrators': [],
            u'subjects': [u'Humanities'],
            u'translators': [],
            u'keywords': [u'Food', u'デザート', u'Pudding'],
            u'title': u'チョコレート',
            u'license_text': u'CC-By 4.0',
            u'license_url': u'http://creativecommons.org/licenses/by/4.0/',
            # 'version': 'draft',
            u'language': None,
            u'print_style': None,
            u'cnx-archive-uri': None,
            u'derived_from_title': None,
            u'derived_from_uri': None,
            }

        fruity = desserts[0]
        self.assertEqual('TranslucentBinder', fruity.__class__.__name__)
        self.assertEqual('Fruity', fruity.metadata['title'])

        apple = fruity[0]
        self.assertEqual('Document', apple.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Apple'
        apple_metadata = apple.metadata.copy()
        summary = etree.fromstring(apple_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, apple_metadata)

        lemon = fruity[1]
        self.assertEqual('Document', lemon.__class__.__name__)
        metadata = base_metadata.copy()
        metadata['title'] = 'Lemon'
        lemon_metadata = lemon.metadata.copy()
        summary = etree.fromstring(lemon_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        self.assertEqual(metadata, lemon_metadata)

        citrus = fruity[2]
        self.assertEqual('TranslucentBinder', citrus.__class__.__name__)
        self.assertEqual(citrus.metadata['title'], 'Citrus')

        self.assertEqual(lemon.metadata, citrus[0].metadata)

        chocolate = desserts[1]
        self.assertEqual('Document', chocolate.__class__.__name__)
        chocolate_metadata = chocolate.metadata.copy()
        summary = etree.fromstring(chocolate_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = u'チョコレート'
        self.assertEqual(metadata, chocolate_metadata)

        extra = desserts[2]
        self.assertEqual('CompositeDocument', extra.__class__.__name__)
        extra_metadata = extra.metadata.copy()
        summary = etree.fromstring(extra_metadata.pop('summary'))
        self.assertEqual('{http://www.w3.org/1999/xhtml}p', summary.tag)
        self.assertEqual('summary', summary.text)
        metadata = base_metadata.copy()
        metadata['title'] = 'Extra Stuff'
        self.assertEqual(metadata, extra_metadata)

class CollateTestCase(BaseModelTestCase):

    @property
    def target(self):
        from cnxepub.collation import collate
        return collate

    def test(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"
document one
",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"
document two
",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b" ")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        def mock_easybake(ruleset, in_html, out_html):
            from lxml import etree
            html = etree.parse(in_html)
            # Add in a composite-page with title "Composite One" here.
            body = html.getroot().xpath(
                '//xhtml:body',
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            comp_elm = etree.SubElement(body, 'div')
            comp_elm.attrib['data-type'] = 'composite-page'
            comp_elm.append(etree.fromstring("""

Composite One

                By:
                Edited by:
                Illustrated by:
                Translated by:

                Published By:

                Licensed:

"""))
            etree.SubElement(comp_elm, 'p').text = "composite document"
            # Add the composite-page to the table-of-contents.
            toc = html.getroot().xpath(
                "//xhtml:*[@id='toc']/xhtml:ol",
                namespaces={'xhtml': 'http://www.w3.org/1999/xhtml'})[0]
            etree.SubElement(toc, 'li').append(etree.fromstring('Composite One'))
            out_html.write(etree.tostring(html))

        with mock.patch('cnxepub.collation.easybake') as easybake:
            easybake.side_effect = mock_easybake
            collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].id, None)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Composite One')

    def test_without_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        result = self.target(binder)
        self.assertIs(binder, result)

    def test_with_ruleset(self):
        binder = self.make_binder(
            '8d75ea29',
            metadata={'version': '3', 'title': "Book One"},
            nodes=[
                self.make_document(
                    id="e78d4f90",
                    content=b"document one",
                    metadata={'version': '3',
                              'title': "Document One"}),
                self.make_document(
                    id="3c448dc6",
                    content=b"document two",
                    metadata={'version': '1',
                              'title': "Document Two"})])

        # Append a ruleset to the binder.
        ruleset = io.BytesIO(b"""\
div[data-type='page'] > div[data-type='metadata'] {
  copy-to: eob-all
}
div[data-type='page'] span {
  copy-to: eob-all
}
body::after {
  content: pending(eob-all);
  class: end-of-book;
  data-type: composite-page;
  container: div;
}

/* copied from cte books/rulesets/common/toc.less */
body > div[data-type="page"],
body > div[data-type="composite-page"]:pass(20) {
  string-set: page-id attr(id);
}
body > div[data-type="page"] > div[data-type="metadata"] > \
    h1[data-type='document-title'],
body > div[data-type="composite-page"] > div[data-type="metadata"] > \
        h1[data-type='document-title']:pass(20) {
  copy-to: page-title;
}
body > div[data-type="page"]::after,
body > div[data-type="composite-page"]:pass(20)::after {
  content: pending(page-title);
  attr-href: "#" string(page-id);
  container: a;
  move-to: page-link;
}
body > div[data-type="page"]::after,
body > div[data-type="composite-page"]:pass(20)::after {
  content: pending(page-link);
  move-to: eob-toc;
  container: li;
}
nav#toc:pass(30) {
  content: '';
}
nav#toc:pass(30)::after {
  content: pending(eob-toc);
  container: ol;
}
""")
        resource = self.make_resource('ruleset', ruleset, 'text/css',
                                      filename='ruleset.css')
        binder.resources.append(resource)

        collated_binder = self.target(binder)

        # Check for the appended composite document
        self.assertEqual(len(collated_binder), 3)
        self.assertEqual(collated_binder[2].metadata['title'],
                         'Document One')
<|endoftext|>


<commit_before><commit_msg>made changes<commit_after><|endoftext|>


<commit_before># TODO
# * make components comparison case-insensitive: NRF51822 == nRF51822
# * make use of os.join operations
# * get rid of jsons - deal directly with csv's
# * strip out special characthers like hyphens
# * make partially-matching components match: NRF51822-QFAA-T == nRF51822
# * get rid of try-excepts
# * add quantity field so that if there's 10x the same part - it should have higher coverage rating
# make two lists - parts that match and parts that don't
# parse 10 more boms
# comment the code better
# make a googledoc with the results

import os
import io
import json
import csv
import pprint

def clean_mpn(mpn):
    garbage_chars = '~!@#$%^&*()_+-={}[]|:;"<>,./?\\\' '
    # proper_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890'
    for i in garbage_chars:
        if i in mpn:
            mpn = mpn.replace(i, '')
    return mpn

def csv2dict(input_csv_file):
    """ read csv file from disk and parse it into a dictionary
        data structure implemented here is a list of dictionaries
        [
            {'mpn':'...', 'brand':'...'},
            {...},
            {...},
            ...
        ]

        refactor (if len(value) > 0 and len(key) > 0:) line
        refactor (if ('mpn' in entry.keys()):) line

        read the docs on DictReader
    """
    print ('parsing %s' % input_csv_file)
    # different strings for the same field in different csv boms
    mpn_csv_fields = ['mpn', 'manufactuer part number', 'part number', 'part#']
    brand_csv_fields = ['manufacturer', 'manufactuer name', 'manu']
    quantity_csv_fields = ['qty', 'quantity']

    with io.open(input_csv_file, 'r', encoding='utf-8') as f:
        dict_from_csv = []
        csv_reader = csv.DictReader(f)
        for line_item in csv_reader:
            entry = {}
            for item in line_item:
                key = str(item).strip()
                value = str(line_item[item]).strip()
                if len(value) > 0 and len(key) > 0:
                    if key.lower() in mpn_csv_fields:
                        # cleaning up special symbols
                        entry['mpn'] = clean_mpn(value)
                    if key.lower() in brand_csv_fields:
                        entry['brand'] = value
                    if key.lower() in quantity_csv_fields:
                        entry['qty'] = value

            # make sure we don't add parts with empty MPN's
            if ('mpn' in entry.keys()):
                # if part was mentioned in the list, but no quantity specified
                # we assume it is used once in the BOM
                if ('qty' not in entry.keys()):
                    entry['qty'] = 1
                dict_from_csv.append(entry)

    return dict_from_csv

def match_boms(reference_bom, target_bom):
    """

    bom_01 and bom_02 are lists of dictionaries with [{'brand':'...', 'mpn':'...' }] format
    find matches between two boms

    part coverage is calculated as
    (sum( part * qty )) / ()

    match examples:
        c: 1n4148  r: 1n4148ws7f
        c: ft232rl  r: ft232rlreel
        c: mmbt3904  r: mmbt3904lt1g
        c: nrf51822  r: nrf51822qfaat
    """

    matches = []
    non_matches = []
    match_part_count = 0
    total_part_count = 0
    coverage = 0.0

    # iterate over the parts in the target BOM
    for target_part in target_bom:
        # make a total part count for the BOM
        total_part_count += int(target_part['qty'])
        for reference_part in reference_bom:

            c = target_part['mpn'].upper()
            r = reference_part['mpn'].upper()

            # check for over- and under-specified MPN's in both lists
            if (c in r or r in c):
                # if match found - put it to the matched list and increase
                # match counter
                matches.append(target_part)
                match_part_count += int(target_part['qty'])
            else:
                # if no match found - put it to the non-match list
                if target_part not in non_matches:
                    non_matches.append(target_part)

    # calculate coverage from total part count in the BOM
    coverage = float(match_part_count / total_part_count)

    return matches, non_matches, coverage, total_part_count

if __name__ == ('__main__'):

    pp = pprint.PrettyPrinter(indent=4)
    """ open csv file, parse it into a dict and save as a json """

    data_folder = '../data/boms/'

    seeedOPL_CSV_file = os.path.join(data_folder, 'csv/OPL list20141222.csv')
    seeed_opl = csv2dict(seeedOPL_CSV_file)

    octopartCPL_CSV_file = os.path.join(
        data_folder, 'csv/Common Parts Library BOM.csv')
    octopart_cpl = csv2dict(octopartCPL_CSV_file)

    arm_BOM_CSV_file = os.path.join(data_folder, 'csv/arm-pro-mini-bom.csv')
    arm_BOM = csv2dict(arm_BOM_CSV_file)

    arduino_BOM_CSV_file = data_folder + 'csv/arduino_bom.csv'
    arduino_BOM = csv2dict(arduino_BOM_CSV_file)

    # print ("================================================================")
    # print ('octopart: ' + str(len(octopart_cpl)) + ' elements')
    # print ("----------------------------------------------------------------")
    # print (octopart_cpl)

    # print ("================================================================")
    # print ('seeed: ' + str(len(seeed_opl)) + ' elements')
    # print ("----------------------------------------------------------------")
    # print (seeed_opl)

    # print ("================================================================")
    # print ('arm: ' + str(len(arm_BOM)) + ' elements')
    # print ("----------------------------------------------------------------")
    # print (arm_BOM)

    # print ("================================================================")
    # print ('arduino: ' + str(len(arduino_BOM)) + ' elements')
    # print ("----------------------------------------------------------------")
    # print (arduino_BOM)

    # Match three boms against CPL
    arm_cpl_matches, arm_cpl_non_matches, arm_cpl_coverage, arm_total_parts = \
        match_boms(octopart_cpl, arm_BOM)
    seeed_cpl_matches, seeed_cpl_non_mathces, seeed_cpl_coverage, seeed_total_parts = \
        match_boms(octopart_cpl, seeed_opl)
    arduino_cpl_matches, arduino_cpl_non_matches, arduino_cpl_coverage, arduino_total_parts = \
        match_boms(octopart_cpl, arduino_BOM)

    arm_opl_matches, arm_opl_non_matches, arm_opl_coverage, arm_total_parts = \
        match_boms(seeed_opl, arm_BOM)
    octopart_opl_matches, octopart_opl_non_matches, octopart_opl_coverage, octopart_total_parts = \
        match_boms(seeed_opl, octopart_cpl)
    arduino_opl_matches, arduino_opl_non_matches, arduino_opl_coverage, arduino_total_parts = \
        match_boms(seeed_opl, arduino_BOM)

    print ("================================================================")
    print ("================  OCTOPART CPL MATCHES  ========================")
    print ("================================================================")

    print ('ArmProMini: covered {:.2f} percent of total {} parts'.format(
           (arm_cpl_coverage * 100), str(arm_total_parts)))

    print ([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty'])
            for i in arm_cpl_matches])
    # pp.pprint(arduino_cpl_non_matches)
    print ('----------------------------------')

    print ('Arduino: covered {:.2f} percent of total {} parts'.format(
           (arduino_cpl_coverage * 100), str(arduino_total_parts)))
    print ([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty'])
            for i in arduino_cpl_matches])
    print ('----------------------------------')

    # print ('Seeed OPL: covered {:.2f} percent of total {} parts'.format(
    #        (seeed_cpl_coverage * 100), str(seeed_total_parts)))
    # print([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty'])
    #        for i in seeed_cpl_matches])
    # print ('----------------------------------')

    print ("================================================================")
    print ("=================    SEEED OPL MATCHES    ======================")
    print ("================================================================")
    print ('ArmProMini: covered {:.2f} percent of total {} parts'.format(
           (arm_opl_coverage * 100), str(arm_total_parts)))
    print([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty']) for i in arm_opl_matches
           if 'brand' and 'mpn' in i])
    print ('----------------------------------')
    print ('Arduino: covered {:.2f} percent of total {} parts'.format(
           (arduino_opl_coverage * 100), str(arduino_total_parts)))
    print([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty']) for i in arduino_opl_matches
           if 'brand' and 'mpn' in i])
    print ('----------------------------------')
    # print ('Octopart CPL: covered {:.2f} percent of total {} parts'.format(
    #        (octopart_opl_coverage * 100), str(octopart_total_parts)))
    # print([i['mpn'] + ' : ' + i['brand'] + ' : x' + str(i['qty']) for i in octopart_opl_matches
    #        if 'brand' and 'mpn' in i])
    # print ('----------------------------------')
<commit_msg>refactor the loops and conditions<commit_after># TODO
# * make components comparison case-insensitive: NRF51822 == nRF51822
# * make use of os.join operations
# * get rid of jsons - deal directly with csv's
# * strip out special characthers like hyphens
# * make partially-matching components match: NRF51822-QFAA-T == nRF51822
# * get rid of try-excepts
# * add quantity field so that if there's 10x the same part - it should have higher coverage rating
# * make two lists - parts that match and parts that don't
# * refactor (if len(value) > 0 and len(key) > 0:) line
# * refactor (if ('mpn' in entry.keys()):) line

# parse 10 more boms
# comment the code better
# make a googledoc with the results

import os
import io
import pprint
import re
import json
import csv

DATA_FOLDER = '../data/boms/'

# CSV file paths
SEEED_FILE = os.path.join(DATA_FOLDER, 'csv/OPL list20141222.csv')
OCTOPART_FILE = os.path.join(DATA_FOLDER, 'csv/Common Parts Library BOM.csv')
ARDUINO_FILE = os.path.join(DATA_FOLDER, 'csv/arduino_bom.csv')
ARM_FILE = os.path.join(DATA_FOLDER, 'csv/arm-pro-mini-bom.csv')

def clean_mpn(mpn):
    ''' remove non-alphanumeric characters from MPN field '''
    garbage_chars = '~!@#$%^&*()_+-={}[]|:;"<>,./?\\\' '

    for i in garbage_chars:
        if i in mpn:
            mpn = mpn.replace(i, '')
    return mpn

def csv2dict(input_csv_file):
    ''' read csv file from disk and parse it into an array of dictionaries
        [{'mpn':'...', 'brand':'...', 'qty':'...'},{...},...,{...}]
    '''
    print ('parsing {}'.format(input_csv_file))
    # different strings for the same field in different csv boms
    MPN_FIELDS = ['mpn', 'manufactuer part number',
        'manufacturer part number', 'part number', 'part#', 'part #']
    BRAND_FIELDS = ['manufacturer', 'manufactuer name', 'manu', 'man']
    QTY_FIELDS = ['qty', 'quantity']

    with io.open(input_csv_file, 'r', encoding='utf-8') as f:
        dict_from_csv = []
        csv_reader = csv.DictReader(f)
        for line_item in csv_reader:
            entry = {}
            for item in line_item:
                key = item.strip().lower()
                value = line_item[item].strip()
                # sort the randomly-named csv entries into a neat dictionary
                if value:
                    if key in MPN_FIELDS: entry['mpn'] = clean_mpn(value)
                    elif key in BRAND_FIELDS: entry['brand'] = value
                    elif key in QTY_FIELDS: entry['qty'] = value

            # if part was mentioned in the list, but no quantity specified
            # we assume it is used once
            if 'qty' not in entry.keys(): entry['qty'] = 1

            # we only need parts with MPN explicitly stated
            if 'mpn' in entry.keys(): dict_from_csv.append(entry)

    return dict_from_csv

def match_boms(reference_bom, target_bom):
    '''
    part coverage is calculated as:
    total count of all matched parts / total count of parts in BOM

    over- and under-specified MPN's are matched
    match examples:
        c: 1n4148  r: 1n4148ws7f
        c: ft232rl  r: ft232rlreel
        c: mmbt3904  r: mmbt3904lt1g
        c: nrf51822  r: nrf51822qfaat
    '''

    # matching results data structure
    results = {
        'matches':[],
        'matches_count':0,
        'non-matches':[],
        'coverage':0,
        'total_parts':0
    }

    # iterate over the parts in the target BOM
    for target_part in target_bom:
        # make a total part count for the BOM
        results['total_parts'] += int(target_part['qty'])

        for reference_part in reference_bom:

            # store mpns in separate variables for concise comparison
            t = target_part['mpn'].upper()
            r = reference_part['mpn'].upper()

            # check for over- and under-specified MPN's in both lists
            if (t in r or r in t):
                # if match found - put it to the matched list
                results['matches'].append(target_part)
            else:
                # if no match found - put it to the non-match list
                if target_part not in results['non-matches']:
                    results['non-matches'].append(target_part)

    # calculate coverage from total part count in the BOM

    parts_covered = sum([int(i['qty']) for i in results['matches']])
    results['coverage'] =  parts_covered / results['total_parts']

    return results

def print_results(name, matches):
    ''' print detailed results of matching procedure '''

    pp = pprint.PrettyPrinter(indent=4)

    print ('{}: covered {:.2f} percent from the total of {} parts'
        .format(
            name,
            matches['coverage'] * 100,
            str(matches['total_parts'])
           )
        )
    print ('>>>> matched parts >>>>')
    pp.pprint(matches['matches'])
    print ('>>>> not matched parts >>>>')
    pp.pprint(matches['non-matches'])
    print ('----------------------------------')

if __name__ == ('__main__'):

    # read files into dictionaries
    seeed_opl = csv2dict(SEEED_FILE)
    octopart_cpl = csv2dict(OCTOPART_FILE)

    arm_BOM = csv2dict(ARM_FILE)
    arduino_BOM = csv2dict(ARDUINO_FILE)

    arm_IN_cpl = match_boms(octopart_cpl, arm_BOM)
    seeed_IN_cpl = match_boms(octopart_cpl, seeed_opl)
    arduino_IN_cpl = match_boms(octopart_cpl, arduino_BOM)

    arm_IN_opl = match_boms(seeed_opl, arm_BOM)
    cpl_IN_opl = match_boms(seeed_opl, octopart_cpl)
    arduino_IN_opl = match_boms(seeed_opl, arduino_BOM)

    print ("================================================================")
    print ("================  OCTOPART CPL MATCHES  ========================")
    print ("================================================================")
    print ()

    print_results('ArmProMini in CPL', arm_IN_cpl)
    print_results('Arduino in CPL', arduino_IN_cpl)
    print_results('Seeed OPL in CPL', seeed_IN_cpl)

    print ("================================================================")
    print ("=================    SEEED OPL MATCHES    ======================")
    print ("================================================================")
    print ()

    print_results('ArmProMini in OPL', arm_IN_opl)
    print_results('Arduino in OPL', arduino_IN_opl)
    print_results('Octopart CPL in OPL', cpl_IN_opl)

<|endoftext|>


<commit_before>ANTIBODIES = [
    {'name': 'Antibody1'},
    {'name': 'Antibody2'},
    {'name': 'Antibody3'},
]

def load_antibodies(testapp):
    for item in ANTIBODIES:
        testapp.post_json('/antibodies/', item, status=201)
<commit_msg>Some sample data<commit_after># -*- coding: utf-8 -*-

ORGANISMS = [
    {
    '_links': {
        'self': {'href': '/organisms/human'},
        },
    'organism_name': 'human',
    'genus': 'Homo',
    'species': 'sapiens',
    'taxon_id': 9606,
    'strain': None,  # Strains and individuals feel like different types
    'individual': None,
    },
]

TARGETS = [
    {
    '_links': {
        'self': {'href': '/targets/tbd'},
        'organism': {'href': '/organisms/human'},
        },
    'target_term_id': '468',
    'target_label': 'ATF4',
    'organism_name': 'human',  # link to organism
    'target_symbol': None,
    'target_gene_name': 'ATF4',
    'target_class': 'generated',
    'aliases': [
        {'alias': 'CREB2', 'source': 'HGNC'},
        {'alias': 'TXREB', 'source': 'HGNC'},
        {'alias': 'CREB-2', 'source': 'HGNC'},
        {'alias': 'TAXREB67', 'source': 'HGNC'},
        ],
    'dbxref': [
        {'db': 'UniProt', 'id': 'P....'},
    ],

    'is_current': True,
    'date_created': '2013-01-17',
    'created_by': 'Myers-Pauli-Behn',
    },
]

SOURCES = [
    {
    '_links': {
        'self': {'href': '/sources/sigma'},
        },
    'source_name': 'sigma',
    'source_title': 'Sigma-Aldrich',
    'url': 'http://www.sigmaaldrich.com',
    },
]

ANTIBODIES = [
    {
    '_links': {
        'self': {'href': '/antibodies/s123'},
        'source': {'href': '/sources/sigma'},
        },
    'antibody_term_id': 'tbd-taxonomic-id',
    'antibody_name': 's123',
    'clonality': 'Monoclonal',
    'host_orgnamism': 'Mouse',
    'source_name': 'Sigma-Aldrich',  # PK
    'product_id': 'WH0000468M1',  # PK
    'lot_id': 'CB191-2B3',  # PK
    'url': 'http://www.sigmaaldrich.com/catalog/product/sigma/wh0000468m1?lang=en&region=US',
    'isotype': u'IgG1κ',
    'purification': None,
    'antibody_description': None,
    'antigen_description': 'ATF4 (NP_001666, a.a. 171-271) partial recombinant protein with GST tag.',
    'antigen_sequence': None,

    'is_obsolete': False,  # Antibody should no longer be used
    'submitted_by': 'Myers-Pauli-Behn',
    'date_created': '2009-01-16',
    'created_by': 'Cherry-Sloan',
    },
]

VALIDATIONS = [
    {
    '_links': {
        'self': {'href': '/antibodies/s123/validations/v456'},
        'antibody': {'href': '/antibodies/sigma'},
        'target': {'href': '/targets/tbd'},
        },
    'method': '',
    'doc': '',
    'caption': '',
    'submitter': '',
    'review_state': '',
    },
]

APPROVALS = [
    {
    '_links': {
        'self': {'href': '/antibodies/Sigma-Aldrich,WH0000468M1,CB191-2B3/approvals/tbd'},
        'antibody': {'href': '/antibodies/Sigma-Aldrich,WH0000468M1,CB191-2B3'},
        'target': {'href': '/targets/tbd'},
        'validations': [],
        },
    },
]

def load_antibodies(testapp):
    for item in ANTIBODIES:
        testapp.post_json('/antibodies/', item, status=201)
<|endoftext|>


<commit_before>"""
Django settings for csinfo project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os
import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ['DJANGO_SECRET_KEY']

# SECURITY WARNING: don't run with debug turned on in production!
if 'DEVELOPMENT' in os.environ and os.environ.get('DEVELOPMENT') == 1:
    DEBUG = True
else:
    DEBUG = False

ALLOWED_HOSTS = [
    '127.0.0.1',
    '0.0.0.0',
    'localhost',
    'ciphersuite.info',
    'ciphersuite-info.herokuapp.com',
]

# Application definition

INSTALLED_APPS = [
    'markdown_deux',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.postgres',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'blog.apps.BlogConfig',
    'sass_processor',
    'directory.apps.DirectoryConfig',
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'csinfo.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                # makes list of static pages globally available
                'directory.context_processors.static_pages'
            ],
        },
    },
]

WSGI_APPLICATION = 'csinfo.wsgi.application'

# Whitenoise storage
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default':  dj_database_url.config()
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'Europe/Berlin'

USE_I18N = True

USE_L10N = True

USE_TZ = True

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
STATIC_URL = '/static/'

# Extra places for collectstatic to find static files.
STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)
<commit_msg>fixes debug switch<commit_after>"""
Django settings for csinfo project.

Generated by 'django-admin startproject' using Django 1.11.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.11/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.11/ref/settings/
"""

import os
import dj_database_url

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = os.environ['DJANGO_SECRET_KEY']

# SECURITY WARNING: don't run with debug turned on in production!
if 'DEVELOPMENT' in os.environ and os.environ.get('DEVELOPMENT', '0') == '1':
    DEBUG = True
else:
    DEBUG = False

ALLOWED_HOSTS = [
    '127.0.0.1',
    '0.0.0.0',
    'localhost',
    'ciphersuite.info',
    'ciphersuite-info.herokuapp.com',
]

# Application definition

INSTALLED_APPS = [
    'markdown_deux',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.postgres',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'blog.apps.BlogConfig',
    'sass_processor',
    'directory.apps.DirectoryConfig',
    'whitenoise.runserver_nostatic',
    'django.contrib.staticfiles',
]

MIDDLEWARE = [
    'django.middleware.security.SecurityMiddleware',
    'whitenoise.middleware.WhiteNoiseMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'csinfo.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                # makes list of static pages globally available
                'directory.context_processors.static_pages'
            ],
        },
    },
]

WSGI_APPLICATION = 'csinfo.wsgi.application'

# Whitenoise storage
# https://warehouse.python.org/project/whitenoise/
STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'

# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

DATABASES = {
    'default':  dj_database_url.config()
}

# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'Europe/Berlin'

USE_I18N = True

USE_L10N = True

USE_TZ = True

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'staticfiles')
STATIC_URL = '/static/'
SASS_PROCESSOR_ENABLED = True

# Extra places for collectstatic to find static files.
STATICFILES_DIRS = (
    os.path.join(PROJECT_ROOT, 'static'),
)
<|endoftext|>


<commit_before>"""This module defines Exporter, a highly configurable converter
that uses Jinja2 to export notebook files into different formats.
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
import io
import os
import inspect
import copy
import collections
import datetime

# other libs/dependencies
from jinja2 import Environment, FileSystemLoader, ChoiceLoader

# IPython imports
from IPython.config.configurable import Configurable
from IPython.config import Config
from IPython.nbformat import current as nbformat
from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict
from IPython.utils.importstring import import_item
from IPython.utils.text import indent
from IPython.utils import py3compat

from IPython.nbconvert import transformers as nbtransformers
from IPython.nbconvert import filters

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

#Jinja2 extensions to load.
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']

default_filters = {
        'indent': indent,
        'markdown': filters.markdown2html,
        'ansi2html': filters.ansi2html,
        'filter_data_type': filters.DataTypeFilter,
        'get_lines': filters.get_lines,
        'highlight': filters.highlight,
        'highlight2html': filters.highlight,
        'highlight2latex': filters.highlight2latex,
        'markdown2latex': filters.markdown2latex,
        'markdown2rst': filters.markdown2rst,
        'pycomment': filters.python_comment,
        'rm_ansi': filters.remove_ansi,
        'rm_dollars': filters.strip_dollars,
        'rm_fake': filters.rm_fake,
        'html_text' : filters.html_text,
        'add_anchor': filters.add_anchor,
        'ansi2latex': filters.ansi2latex,
        'rm_math_space': filters.rm_math_space,
        'wrap': filters.wrap
}

#-----------------------------------------------------------------------------
# Class
#-----------------------------------------------------------------------------

class ResourcesDict(collections.defaultdict):
    def __missing__(self, key):
        return ''

class Exporter(Configurable):
    """
    Exports notebooks into other file formats.  Uses Jinja 2 templating engine
    to output new formats.  Inherit from this class if you are creating a new
    template type along with new filters/transformers.  If the filters/
    transformers provided by default suffice, there is no need to inherit from
    this class.  Instead, override the template_file and file_extension
    traits via a config file.

    {filters}
    """

    # finish the docstring
    __doc__ = __doc__.format(filters = '- '+'\n    - '.join(default_filters.keys()))

    template_file = Unicode(
            '', config=True,
            help="Name of the template file to use")

    file_extension = Unicode(
        'txt', config=True,
        help="Extension of the file that should be written to disk"
        )

    template_path = Unicode(
        os.path.join("..", "templates"), config=True,
        help="Path where the template files are located.")

    template_skeleton_path = Unicode(
        os.path.join("..", "templates", "skeleton"), config=True,
        help="Path where the template skeleton files are located.")

    #Jinja block definitions
    jinja_comment_block_start = Unicode("", config=True)
    jinja_comment_block_end = Unicode("", config=True)
    jinja_variable_block_start = Unicode("", config=True)
    jinja_variable_block_end = Unicode("", config=True)
    jinja_logic_block_start = Unicode("", config=True)
    jinja_logic_block_end = Unicode("", config=True)

    #Extension that the template files use.
    template_extension = Unicode(".tpl", config=True)

    #Configurability, allows the user to easily add filters and transformers.
    transformers = List(config=True,
        help="""List of transformers, by name or namespace, to enable.""")

    filters = Dict(config=True,
        help="""Dictionary of filters, by name and namespace, to add to the Jinja
        environment.""")

    default_transformers = List([nbtransformers.coalesce_streams,
                                 nbtransformers.SVG2PDFTransformer,
                                 nbtransformers.ExtractFigureTransformer,
                                 nbtransformers.CSSHTMLHeaderTransformer,
                                 nbtransformers.RevealHelpTransformer,
                                 nbtransformers.LatexTransformer,
                                 nbtransformers.SphinxTransformer],
        config=True,
        help="""List of transformers available by default, by name, namespace,
        instance, or type.""")

    def __init__(self, config=None, extra_loaders=None, **kw):
        """
        Public constructor

        Parameters
        ----------
        config : config
            User configuration instance.
        extra_loaders : list[of Jinja Loaders]
            ordered list of Jinja loder to find templates. Will be tried in order
            before the default FileSysteme ones.
        """

        #Call the base class constructor
        c = self.default_config
        if config:
            c.merge(config)

        super(Exporter, self).__init__(config=c, **kw)

        #Init
        self._init_environment(extra_loaders=extra_loaders)
        self._init_transformers()
        self._init_filters()

    @property
    def default_config(self):
        return Config()

    def from_notebook_node(self, nb, resources=None, **kw):
        """
        Convert a notebook from a notebook node instance.

        Parameters
        ----------
        nb : Notebook node
        resources : dict (**kw)
            of additional resources that can be accessed read/write by
            transformers and filters.
        """
        nb_copy = copy.deepcopy(nb)
        resources = self._init_resources(resources)

        #Preprocess
        nb_copy, resources = self._transform(nb_copy, resources)

        #Convert
        self.template = self.environment.get_template(self.template_file + self.template_extension)
        output = self.template.render(nb=nb_copy, resources=resources)
        return output, resources

    def from_filename(self, filename, resources=None, **kw):
        """
        Convert a notebook from a notebook file.

        Parameters
        ----------
        filename : str
            Full filename of the notebook file to open and convert.
        """

        #Pull the metadata from the filesystem.
        if resources is None:
            resources = ResourcesDict()
        if not 'metadata' in resources or resources['metadata'] == '':
            resources['metadata'] = ResourcesDict()
        basename = os.path.basename(filename)
        notebook_name = basename[:basename.rfind('.')]
        resources['metadata']['name'] = notebook_name

        modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
        resources['metadata']['modified_date'] = modified_date.strftime("%B %-d, %Y")

        with io.open(filename) as f:
            return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw)

    def from_file(self, file_stream, resources=None, **kw):
        """
        Convert a notebook from a notebook file.

        Parameters
        ----------
        file_stream : file-like object
            Notebook file-like object to convert.
        """
        return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw)

    def register_transformer(self, transformer, enabled=False):
        """
        Register a transformer.
        Transformers are classes that act upon the notebook before it is
        passed into the Jinja templating engine.  Transformers are also
        capable of passing additional information to the Jinja
        templating engine.

        Parameters
        ----------
        transformer : transformer
        """
        if transformer is None:
            raise TypeError('transformer')
        isclass = isinstance(transformer, type)
        constructed = not isclass

        #Handle transformer's registration based on it's type
        if constructed and isinstance(transformer, py3compat.string_types):
            #Transformer is a string, import the namespace and recursively call
            #this register_transformer method
            transformer_cls = import_item(transformer)
            return self.register_transformer(transformer_cls, enabled)

        if constructed and hasattr(transformer, '__call__'):
            #Transformer is a function, no need to construct it.
            #Register and return the transformer.
            if enabled:
                transformer.enabled = True
            self._transformers.append(transformer)
            return transformer

        elif isclass and isinstance(transformer, MetaHasTraits):
            #Transformer is configurable.  Make sure to pass in new default for
            #the enabled flag if one was specified.
            self.register_transformer(transformer(parent=self), enabled)

        elif isclass:
            #Transformer is not configurable, construct it
            self.register_transformer(transformer(), enabled)

        else:
            #Transformer is an instance of something without a __call__
            #attribute.
            raise TypeError('transformer')

    def register_filter(self, name, jinja_filter):
        """
        Register a filter.
        A filter is a function that accepts and acts on one string.
        The filters are accesible within the Jinja templating engine.

        Parameters
        ----------
        name : str
            name to give the filter in the Jinja engine
        filter : filter
        """
        if jinja_filter is None:
            raise TypeError('filter')
        isclass = isinstance(jinja_filter, type)
        constructed = not isclass

        #Handle filter's registration based on it's type
        if constructed and isinstance(jinja_filter, py3compat.string_types):
            #filter is a string, import the namespace and recursively call
            #this register_filter method
            filter_cls = import_item(jinja_filter)
            return self.register_filter(name, filter_cls)

        if constructed and hasattr(jinja_filter, '__call__'):
            #filter is a function, no need to construct it.
            self.environment.filters[name] = jinja_filter
            return jinja_filter

        elif isclass and isinstance(jinja_filter, MetaHasTraits):
            #filter is configurable.  Make sure to pass in new default for
            #the enabled flag if one was specified.
            filter_instance = jinja_filter(parent=self)
            self.register_filter(name, filter_instance )

        elif isclass:
            #filter is not configurable, construct it
            filter_instance = jinja_filter()
            self.register_filter(name, filter_instance)

        else:
            #filter is an instance of something without a __call__
            #attribute.
            raise TypeError('filter')

    def _init_environment(self, extra_loaders=None):
        """
        Create the Jinja templating environment.
        """
        here = os.path.dirname(os.path.realpath(__file__))
        loaders = []
        if extra_loaders:
            loaders.extend(extra_loaders)

        loaders.append(FileSystemLoader([
                os.path.join(here, self.template_path),
                os.path.join(here, self.template_skeleton_path),
                ]))

        self.environment = Environment(
            loader= ChoiceLoader(loaders),
            extensions=JINJA_EXTENSIONS
            )

        #Set special Jinja2 syntax that will not conflict with latex.
        if self.jinja_logic_block_start:
            self.environment.block_start_string = self.jinja_logic_block_start
        if self.jinja_logic_block_end:
            self.environment.block_end_string = self.jinja_logic_block_end
        if self.jinja_variable_block_start:
            self.environment.variable_start_string = self.jinja_variable_block_start
        if self.jinja_variable_block_end:
            self.environment.variable_end_string = self.jinja_variable_block_end
        if self.jinja_comment_block_start:
            self.environment.comment_start_string = self.jinja_comment_block_start
        if self.jinja_comment_block_end:
            self.environment.comment_end_string = self.jinja_comment_block_end

    def _init_transformers(self):
        """
        Register all of the transformers needed for this exporter, disabled
        unless specified explicitly.
        """
        self._transformers = []

        #Load default transformers (not necessarly enabled by default).
        if self.default_transformers:
            for transformer in self.default_transformers:
                self.register_transformer(transformer)

        #Load user transformers.  Enable by default.
        if self.transformers:
            for transformer in self.transformers:
                self.register_transformer(transformer, enabled=True)

    def _init_filters(self):
        """
        Register all of the filters required for the exporter.
        """

        #Add default filters to the Jinja2 environment
        for key, value in default_filters.items():
            self.register_filter(key, value)

        #Load user filters.  Overwrite existing filters if need be.
        if self.filters:
            for key, user_filter in self.filters.items():
                self.register_filter(key, user_filter)

    def _init_resources(self, resources):

        #Make sure the resources dict is of ResourcesDict type.
        if resources is None:
            resources = ResourcesDict()
        if not isinstance(resources, ResourcesDict):
            new_resources = ResourcesDict()
            new_resources.update(resources)
            resources = new_resources

        #Make sure the metadata extension exists in resources
        if 'metadata' in resources:
            if not isinstance(resources['metadata'], ResourcesDict):
                resources['metadata'] = ResourcesDict(resources['metadata'])
        else:
            resources['metadata'] = ResourcesDict()
            if not resources['metadata']['name']:
                resources['metadata']['name'] = 'Notebook'

        #Set the output extension
        resources['output_extension'] = self.file_extension
        return resources

    def _transform(self, nb, resources):
        """
        Preprocess the notebook before passing it into the Jinja engine.
        To preprocess the notebook is to apply all of the

        Parameters
        ----------
        nb : notebook node
            notebook that is being exported.
        resources : a dict of additional resources that
            can be accessed read/write by transformers
            and filters.
        """

        # Do a copy.deepcopy first,
        # we are never safe enough with what the transformers could do.
        nbc =  copy.deepcopy(nb)
        resc = copy.deepcopy(resources)

        #Run each transformer on the notebook.  Carry the output along
        #to each transformer
        for transformer in self._transformers:
            nbc, resc = transformer(nbc, resc)
        return nbc, resc
<commit_msg>Fix for #3719<commit_after>"""This module defines Exporter, a highly configurable converter
that uses Jinja2 to export notebook files into different formats.
"""

#-----------------------------------------------------------------------------
# Copyright (c) 2013, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
import io
import os
import inspect
import copy
import collections
import datetime

# other libs/dependencies
from jinja2 import Environment, FileSystemLoader, ChoiceLoader

# IPython imports
from IPython.config.configurable import Configurable
from IPython.config import Config
from IPython.nbformat import current as nbformat
from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict
from IPython.utils.importstring import import_item
from IPython.utils.text import indent
from IPython.utils import py3compat

from IPython.nbconvert import transformers as nbtransformers
from IPython.nbconvert import filters

#-----------------------------------------------------------------------------
# Globals and constants
#-----------------------------------------------------------------------------

#Jinja2 extensions to load.
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']

default_filters = {
        'indent': indent,
        'markdown': filters.markdown2html,
        'ansi2html': filters.ansi2html,
        'filter_data_type': filters.DataTypeFilter,
        'get_lines': filters.get_lines,
        'highlight': filters.highlight,
        'highlight2html': filters.highlight,
        'highlight2latex': filters.highlight2latex,
        'markdown2latex': filters.markdown2latex,
        'markdown2rst': filters.markdown2rst,
        'pycomment': filters.python_comment,
        'rm_ansi': filters.remove_ansi,
        'rm_dollars': filters.strip_dollars,
        'rm_fake': filters.rm_fake,
        'html_text' : filters.html_text,
        'add_anchor': filters.add_anchor,
        'ansi2latex': filters.ansi2latex,
        'rm_math_space': filters.rm_math_space,
        'wrap': filters.wrap
}

#-----------------------------------------------------------------------------
# Class
#-----------------------------------------------------------------------------

class ResourcesDict(collections.defaultdict):
    def __missing__(self, key):
        return ''

class Exporter(Configurable):
    """
    Exports notebooks into other file formats.  Uses Jinja 2 templating engine
    to output new formats.  Inherit from this class if you are creating a new
    template type along with new filters/transformers.  If the filters/
    transformers provided by default suffice, there is no need to inherit from
    this class.  Instead, override the template_file and file_extension
    traits via a config file.

    {filters}
    """

    # finish the docstring
    __doc__ = __doc__.format(filters = '- '+'\n    - '.join(default_filters.keys()))

    template_file = Unicode(
            '', config=True,
            help="Name of the template file to use")

    file_extension = Unicode(
        'txt', config=True,
        help="Extension of the file that should be written to disk"
        )

    template_path = Unicode(
        os.path.join("..", "templates"), config=True,
        help="Path where the template files are located.")

    template_skeleton_path = Unicode(
        os.path.join("..", "templates", "skeleton"), config=True,
        help="Path where the template skeleton files are located.")

    #Jinja block definitions
    jinja_comment_block_start = Unicode("", config=True)
    jinja_comment_block_end = Unicode("", config=True)
    jinja_variable_block_start = Unicode("", config=True)
    jinja_variable_block_end = Unicode("", config=True)
    jinja_logic_block_start = Unicode("", config=True)
    jinja_logic_block_end = Unicode("", config=True)

    #Extension that the template files use.
    template_extension = Unicode(".tpl", config=True)

    #Configurability, allows the user to easily add filters and transformers.
    transformers = List(config=True,
        help="""List of transformers, by name or namespace, to enable.""")

    filters = Dict(config=True,
        help="""Dictionary of filters, by name and namespace, to add to the Jinja
        environment.""")

    default_transformers = List([nbtransformers.coalesce_streams,
                                 nbtransformers.SVG2PDFTransformer,
                                 nbtransformers.ExtractFigureTransformer,
                                 nbtransformers.CSSHTMLHeaderTransformer,
                                 nbtransformers.RevealHelpTransformer,
                                 nbtransformers.LatexTransformer,
                                 nbtransformers.SphinxTransformer],
        config=True,
        help="""List of transformers available by default, by name, namespace,
        instance, or type.""")

    def __init__(self, config=None, extra_loaders=None, **kw):
        """
        Public constructor

        Parameters
        ----------
        config : config
            User configuration instance.
        extra_loaders : list[of Jinja Loaders]
            ordered list of Jinja loder to find templates. Will be tried in order
            before the default FileSysteme ones.
        """

        #Call the base class constructor
        c = self.default_config
        if config:
            c.merge(config)

        super(Exporter, self).__init__(config=c, **kw)

        #Init
        self._init_environment(extra_loaders=extra_loaders)
        self._init_transformers()
        self._init_filters()

    @property
    def default_config(self):
        return Config()

    def from_notebook_node(self, nb, resources=None, **kw):
        """
        Convert a notebook from a notebook node instance.

        Parameters
        ----------
        nb : Notebook node
        resources : dict (**kw)
            of additional resources that can be accessed read/write by
            transformers and filters.
        """
        nb_copy = copy.deepcopy(nb)
        resources = self._init_resources(resources)

        #Preprocess
        nb_copy, resources = self._transform(nb_copy, resources)

        #Convert
        self.template = self.environment.get_template(self.template_file + self.template_extension)
        output = self.template.render(nb=nb_copy, resources=resources)
        return output, resources

    def from_filename(self, filename, resources=None, **kw):
        """
        Convert a notebook from a notebook file.

        Parameters
        ----------
        filename : str
            Full filename of the notebook file to open and convert.
        """

        #Pull the metadata from the filesystem.
        if resources is None:
            resources = ResourcesDict()
        if not 'metadata' in resources or resources['metadata'] == '':
            resources['metadata'] = ResourcesDict()
        basename = os.path.basename(filename)
        notebook_name = basename[:basename.rfind('.')]
        resources['metadata']['name'] = notebook_name

        modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
        resources['metadata']['modified_date'] = modified_date.strftime("%B %d, %Y")

        with io.open(filename) as f:
            return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw)

    def from_file(self, file_stream, resources=None, **kw):
        """
        Convert a notebook from a notebook file.

        Parameters
        ----------
        file_stream : file-like object
            Notebook file-like object to convert.
        """
        return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw)

    def register_transformer(self, transformer, enabled=False):
        """
        Register a transformer.
        Transformers are classes that act upon the notebook before it is
        passed into the Jinja templating engine.  Transformers are also
        capable of passing additional information to the Jinja
        templating engine.

        Parameters
        ----------
        transformer : transformer
        """
        if transformer is None:
            raise TypeError('transformer')
        isclass = isinstance(transformer, type)
        constructed = not isclass

        #Handle transformer's registration based on it's type
        if constructed and isinstance(transformer, py3compat.string_types):
            #Transformer is a string, import the namespace and recursively call
            #this register_transformer method
            transformer_cls = import_item(transformer)
            return self.register_transformer(transformer_cls, enabled)

        if constructed and hasattr(transformer, '__call__'):
            #Transformer is a function, no need to construct it.
            #Register and return the transformer.
            if enabled:
                transformer.enabled = True
            self._transformers.append(transformer)
            return transformer

        elif isclass and isinstance(transformer, MetaHasTraits):
            #Transformer is configurable.  Make sure to pass in new default for
            #the enabled flag if one was specified.
            self.register_transformer(transformer(parent=self), enabled)

        elif isclass:
            #Transformer is not configurable, construct it
            self.register_transformer(transformer(), enabled)

        else:
            #Transformer is an instance of something without a __call__
            #attribute.
            raise TypeError('transformer')

    def register_filter(self, name, jinja_filter):
        """
        Register a filter.
        A filter is a function that accepts and acts on one string.
        The filters are accesible within the Jinja templating engine.

        Parameters
        ----------
        name : str
            name to give the filter in the Jinja engine
        filter : filter
        """
        if jinja_filter is None:
            raise TypeError('filter')
        isclass = isinstance(jinja_filter, type)
        constructed = not isclass

        #Handle filter's registration based on it's type
        if constructed and isinstance(jinja_filter, py3compat.string_types):
            #filter is a string, import the namespace and recursively call
            #this register_filter method
            filter_cls = import_item(jinja_filter)
            return self.register_filter(name, filter_cls)

        if constructed and hasattr(jinja_filter, '__call__'):
            #filter is a function, no need to construct it.
            self.environment.filters[name] = jinja_filter
            return jinja_filter

        elif isclass and isinstance(jinja_filter, MetaHasTraits):
            #filter is configurable.  Make sure to pass in new default for
            #the enabled flag if one was specified.
            filter_instance = jinja_filter(parent=self)
            self.register_filter(name, filter_instance )

        elif isclass:
            #filter is not configurable, construct it
            filter_instance = jinja_filter()
            self.register_filter(name, filter_instance)

        else:
            #filter is an instance of something without a __call__
            #attribute.
            raise TypeError('filter')

    def _init_environment(self, extra_loaders=None):
        """
        Create the Jinja templating environment.
        """
        here = os.path.dirname(os.path.realpath(__file__))
        loaders = []
        if extra_loaders:
            loaders.extend(extra_loaders)

        loaders.append(FileSystemLoader([
                os.path.join(here, self.template_path),
                os.path.join(here, self.template_skeleton_path),
                ]))

        self.environment = Environment(
            loader= ChoiceLoader(loaders),
            extensions=JINJA_EXTENSIONS
            )

        #Set special Jinja2 syntax that will not conflict with latex.
        if self.jinja_logic_block_start:
            self.environment.block_start_string = self.jinja_logic_block_start
        if self.jinja_logic_block_end:
            self.environment.block_end_string = self.jinja_logic_block_end
        if self.jinja_variable_block_start:
            self.environment.variable_start_string = self.jinja_variable_block_start
        if self.jinja_variable_block_end:
            self.environment.variable_end_string = self.jinja_variable_block_end
        if self.jinja_comment_block_start:
            self.environment.comment_start_string = self.jinja_comment_block_start
        if self.jinja_comment_block_end:
            self.environment.comment_end_string = self.jinja_comment_block_end

    def _init_transformers(self):
        """
        Register all of the transformers needed for this exporter, disabled
        unless specified explicitly.
        """
        self._transformers = []

        #Load default transformers (not necessarly enabled by default).
        if self.default_transformers:
            for transformer in self.default_transformers:
                self.register_transformer(transformer)

        #Load user transformers.  Enable by default.
        if self.transformers:
            for transformer in self.transformers:
                self.register_transformer(transformer, enabled=True)

    def _init_filters(self):
        """
        Register all of the filters required for the exporter.
        """

        #Add default filters to the Jinja2 environment
        for key, value in default_filters.items():
            self.register_filter(key, value)

        #Load user filters.  Overwrite existing filters if need be.
        if self.filters:
            for key, user_filter in self.filters.items():
                self.register_filter(key, user_filter)

    def _init_resources(self, resources):

        #Make sure the resources dict is of ResourcesDict type.
        if resources is None:
            resources = ResourcesDict()
        if not isinstance(resources, ResourcesDict):
            new_resources = ResourcesDict()
            new_resources.update(resources)
            resources = new_resources

        #Make sure the metadata extension exists in resources
        if 'metadata' in resources:
            if not isinstance(resources['metadata'], ResourcesDict):
                resources['metadata'] = ResourcesDict(resources['metadata'])
        else:
            resources['metadata'] = ResourcesDict()
            if not resources['metadata']['name']:
                resources['metadata']['name'] = 'Notebook'

        #Set the output extension
        resources['output_extension'] = self.file_extension
        return resources

    def _transform(self, nb, resources):
        """
        Preprocess the notebook before passing it into the Jinja engine.
        To preprocess the notebook is to apply all of the

        Parameters
        ----------
        nb : notebook node
            notebook that is being exported.
        resources : a dict of additional resources that
            can be accessed read/write by transformers
            and filters.
        """

        # Do a copy.deepcopy first,
        # we are never safe enough with what the transformers could do.
        nbc =  copy.deepcopy(nb)
        resc = copy.deepcopy(resources)

        #Run each transformer on the notebook.  Carry the output along
        #to each transformer
        for transformer in self._transformers:
            nbc, resc = transformer(nbc, resc)
        return nbc, resc
<|endoftext|>


<commit_before>import json, os, random, re, time

from CommandTemplate import CommandTemplate
import GlobalStore
import SharedFunctions
from IrcMessage import IrcMessage

class Command(CommandTemplate):
	triggers = ['startrektip', 'startrektips', 'sttip', 'sttips']
	helptext = "Shows a tip from the provided Star Trek character, or a random one. Add a (regex) search after the name to search for a specific tip"
	scheduledFunctionTime = 21600.0  #Six hours in seconds

	twitterUsernames = {'data': 'Data_Tips', 'guinan': 'GuinanTips', 'laforge': 'LaForgeTips', 'locutus': 'LocutusTips', 'picard': 'PicardTips',
						'quark': 'QuarkTips', 'riker': 'RikerTips', 'rikergoogling': 'RikerGoogling', 'worf': 'WorfTips', 'worfemail': 'WorfEmail'}
	# Not all 'tips' are actually tips. This is a list of a replacement term to use if 'tip' is not accurate. It replaces the entire part before the colon
	resultPrefix = {'rikergoogling': "Riker searched", 'worfemail': "Worf's Outbox"}
	isUpdating = False

	def onLoad(self):
		GlobalStore.commandhandler.addCommandFunctions(__file__, 'getStarTrekTip', self.getTip)
		#Add the available names to the helptext
		self.helptext += ". Available names are: " + ", ".join(sorted(self.twitterUsernames.keys()))

	def execute(self, message):
		"""
		:type message: IrcMessage
		"""

		if self.isUpdating:
			message.reply("Sorry, but I'm updating my data (hah) at the moment. Try again in a bit!", "say")
			return

		name = "random"
		if message.messagePartsLength > 0:
			name = message.messageParts[0].lower()
		if name == 'update':
			if self.scheduledFunctionIsExecuting:
				message.reply("I'm already updating!", "say")
			else:
				message.reply("Ok, I'll update my list of Star Trek Tips. But since they have to come from the future, it might take a while. Try again in, oh, half a minute or so, just to be sure", "say")
				self.executeScheduledFunction()
				self.resetScheduledFunctionGreenlet()
			return
		searchterm = None if message.messagePartsLength <= 1 else " ".join(message.messageParts[1:])
		message.reply(self.getTip(name, searchterm), "say")

	def getTweets(self, name='random', searchterm=None):
		name = name.lower()
		if name == 'random':
			name = random.choice(self.twitterUsernames.keys())
		if name not in self.twitterUsernames:
			return (False, "I don't know anybody by the name of '{}', sorry. ".format(name))
		tweetFileName = os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', '{}.txt'.format(self.twitterUsernames[name]))
		if not os.path.exists(tweetFileName):
			self.executeScheduledFunction()
			return (False, "I don't seem to have the tweets for '{}', sorry! I'll retrieve them right away, try again in a bit".format(name))
		tweets = SharedFunctions.getAllLinesFromFile(tweetFileName)
		if searchterm is not None:
			#Search terms provided! Go through all the tweets to find matches
			regex = None
			try:
				regex = re.compile(searchterm, re.IGNORECASE)
			except (re.error, SyntaxError):
				self.logWarning("[STtip] '{}' is an invalid regular expression. Using it literally".format(searchterm))
			for i in xrange(0, len(tweets)):
				#Take a tweet from the start, and only put it back at the end if it matches the regex
				tweet = tweets.pop(0)
				if regex and regex.search(tweet) or searchterm in tweet:
					tweets.append(tweet)
		if len(tweets) == 0:
			return (False, "Sorry, no tweets matching your search were found")
		else:
			return (True, tweets)

	def getTip(self, name='random', searchterm=None):
		name = name.lower()
		if name == 'random':
			name = random.choice(self.twitterUsernames.keys())
		getTweetsReply = self.getTweets(name, searchterm)
		if not getTweetsReply[0]:
			return getTweetsReply[1]
		else:
			tweets = getTweetsReply[1]
			tweetCount = len(tweets)
			replytext = random.choice(tweets).strip()
			#Always make sure the result starts with "[name] tip: "
			if not replytext.lower().startswith(name):
				#Get the special prefix, if any. Otherwise, just do the default "[name] tip: "
				tipPrefix = self.resultPrefix.get(name, u"{} tip".format(name.capitalize()))
				replytext = u"{}: {}".format(tipPrefix, replytext)
			#Only add a tweet count if a search term was provided and there's more than one
			if searchterm is not None and tweetCount > 1:
				replytext += u" [{:,} more matching tweet{}]".format(tweetCount - 1, u's' if tweetCount > 2 else u'')
			replytext = replytext.encode('utf-8', 'replace')
			return replytext

	def executeScheduledFunction(self):
		starttime = time.time()
		self.isUpdating = True
		#First load all the stored tweet data, if it exists
		twitterInfoFilename = os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', 'metadata.json')
		if os.path.exists(twitterInfoFilename):
			with open(twitterInfoFilename, 'r') as twitterInfoFile:
				storedInfo = json.load(twitterInfoFile)
		else:
			storedInfo = {}
			#Create the 'tweets' folder if it doesn't exist already, so we can create our files in there once we're done
			if not os.path.exists(os.path.dirname(twitterInfoFilename)):
				os.makedirs(os.path.dirname(twitterInfoFilename))
		#Go through all the names we need to update
		for name, username in self.twitterUsernames.iteritems():
			highestIdDownloaded = 0
			if username not in storedInfo:
				storedInfo[username] = {'linecount': 0}
			elif "highestIdDownloaded" in storedInfo[username]:
				highestIdDownloaded = storedInfo[username]['highestIdDownloaded']
			tweetResponse = SharedFunctions.downloadTweets(username, downloadNewerThanId=highestIdDownloaded)
			if not tweetResponse[0]:
				self.logError("[STTip] Something went wrong while downloading new tweets for '{}', skipping".format(username))
				continue
			#If there aren't any tweets, stop here
			if len(tweetResponse[1]) == 0:
				continue
			tweets = tweetResponse[1]
			#Reverse tweets so they're from old to new. That way when we write them to file, the entire file will be old to new
			# Not necessary but neat
			tweets.reverse()
			#All tweets downloaded. Time to process them
			tweetfile = open(os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', "{}.txt".format(username)), "a")
			for tweet in tweets:
				tweetfile.write(tweet['text'].replace('\n', ' ').encode(encoding='utf-8', errors='replace') + '\n')
			tweetfile.close()
			#Get the id of the last tweet in the list (the newest one), so we know where to start downloading from next time
			storedInfo[username]['highestIdDownloaded'] = tweets[-1]['id']
			storedInfo[username]['linecount'] += len(tweets)

		#Save the stored info to disk too, for future lookups
		with open(twitterInfoFilename, 'w') as twitterFile:
			twitterFile.write(json.dumps(storedInfo))

		self.isUpdating = False
		self.logInfo("[STtip] Updating tweets took {} seconds".format(time.time() - starttime))
<commit_msg>[STtip] use CommandTemplate's update boolean<commit_after>import json, os, random, re, time

from CommandTemplate import CommandTemplate
import GlobalStore
import SharedFunctions
from IrcMessage import IrcMessage

class Command(CommandTemplate):
	triggers = ['startrektip', 'startrektips', 'sttip', 'sttips']
	helptext = "Shows a tip from the provided Star Trek character, or a random one. Add a (regex) search after the name to search for a specific tip"
	scheduledFunctionTime = 21600.0  #Six hours in seconds

	twitterUsernames = {'data': 'Data_Tips', 'guinan': 'GuinanTips', 'laforge': 'LaForgeTips', 'locutus': 'LocutusTips', 'picard': 'PicardTips',
						'quark': 'QuarkTips', 'riker': 'RikerTips', 'rikergoogling': 'RikerGoogling', 'worf': 'WorfTips', 'worfemail': 'WorfEmail'}
	# Not all 'tips' are actually tips. This is a list of a replacement term to use if 'tip' is not accurate. It replaces the entire part before the colon
	resultPrefix = {'rikergoogling': "Riker searched", 'worfemail': "Worf's Outbox"}

	def onLoad(self):
		GlobalStore.commandhandler.addCommandFunctions(__file__, 'getStarTrekTip', self.getTip)
		#Add the available names to the helptext
		self.helptext += ". Available names are: " + ", ".join(sorted(self.twitterUsernames.keys()))

	def execute(self, message):
		"""
		:type message: IrcMessage
		"""

		if self.scheduledFunctionIsExecuting:
			message.reply("Sorry, but I'm updating my data (hah) at the moment. Try again in a bit!", "say")
			return

		name = "random"
		if message.messagePartsLength > 0:
			name = message.messageParts[0].lower()
		if name == 'update':
			if self.scheduledFunctionIsExecuting:
				message.reply("I'm already updating!", "say")
			else:
				message.reply("Ok, I'll update my list of Star Trek Tips. But since they have to come from the future, it might take a while. Try again in, oh, half a minute or so, just to be sure", "say")
				self.executeScheduledFunction()
				self.resetScheduledFunctionGreenlet()
			return
		searchterm = None if message.messagePartsLength <= 1 else " ".join(message.messageParts[1:])
		message.reply(self.getTip(name, searchterm), "say")

	def getTweets(self, name='random', searchterm=None):
		name = name.lower()
		if name == 'random':
			name = random.choice(self.twitterUsernames.keys())
		if name not in self.twitterUsernames:
			return (False, "I don't know anybody by the name of '{}', sorry. ".format(name))
		tweetFileName = os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', '{}.txt'.format(self.twitterUsernames[name]))
		if not os.path.exists(tweetFileName):
			self.executeScheduledFunction()
			return (False, "I don't seem to have the tweets for '{}', sorry! I'll retrieve them right away, try again in a bit".format(name))
		tweets = SharedFunctions.getAllLinesFromFile(tweetFileName)
		if searchterm is not None:
			#Search terms provided! Go through all the tweets to find matches
			regex = None
			try:
				regex = re.compile(searchterm, re.IGNORECASE)
			except (re.error, SyntaxError):
				self.logWarning("[STtip] '{}' is an invalid regular expression. Using it literally".format(searchterm))
			for i in xrange(0, len(tweets)):
				#Take a tweet from the start, and only put it back at the end if it matches the regex
				tweet = tweets.pop(0)
				if regex and regex.search(tweet) or searchterm in tweet:
					tweets.append(tweet)
		if len(tweets) == 0:
			return (False, "Sorry, no tweets matching your search were found")
		else:
			return (True, tweets)

	def getTip(self, name='random', searchterm=None):
		name = name.lower()
		if name == 'random':
			name = random.choice(self.twitterUsernames.keys())
		getTweetsReply = self.getTweets(name, searchterm)
		if not getTweetsReply[0]:
			return getTweetsReply[1]
		else:
			tweets = getTweetsReply[1]
			tweetCount = len(tweets)
			replytext = random.choice(tweets).strip()
			#Always make sure the result starts with "[name] tip: "
			if not replytext.lower().startswith(name):
				#Get the special prefix, if any. Otherwise, just do the default "[name] tip: "
				tipPrefix = self.resultPrefix.get(name, u"{} tip".format(name.capitalize()))
				replytext = u"{}: {}".format(tipPrefix, replytext)
			#Only add a tweet count if a search term was provided and there's more than one
			if searchterm is not None and tweetCount > 1:
				replytext += u" [{:,} more matching tweet{}]".format(tweetCount - 1, u's' if tweetCount > 2 else u'')
			replytext = replytext.encode('utf-8', 'replace')
			return replytext

	def executeScheduledFunction(self):
		starttime = time.time()
		self.isUpdating = True
		#First load all the stored tweet data, if it exists
		twitterInfoFilename = os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', 'metadata.json')
		if os.path.exists(twitterInfoFilename):
			with open(twitterInfoFilename, 'r') as twitterInfoFile:
				storedInfo = json.load(twitterInfoFile)
		else:
			storedInfo = {}
			#Create the 'tweets' folder if it doesn't exist already, so we can create our files in there once we're done
			if not os.path.exists(os.path.dirname(twitterInfoFilename)):
				os.makedirs(os.path.dirname(twitterInfoFilename))
		#Go through all the names we need to update
		for name, username in self.twitterUsernames.iteritems():
			highestIdDownloaded = 0
			if username not in storedInfo:
				storedInfo[username] = {'linecount': 0}
			elif "highestIdDownloaded" in storedInfo[username]:
				highestIdDownloaded = storedInfo[username]['highestIdDownloaded']
			tweetResponse = SharedFunctions.downloadTweets(username, downloadNewerThanId=highestIdDownloaded)
			if not tweetResponse[0]:
				self.logError("[STTip] Something went wrong while downloading new tweets for '{}', skipping".format(username))
				continue
			#If there aren't any tweets, stop here
			if len(tweetResponse[1]) == 0:
				continue
			tweets = tweetResponse[1]
			#Reverse tweets so they're from old to new. That way when we write them to file, the entire file will be old to new
			# Not necessary but neat
			tweets.reverse()
			#All tweets downloaded. Time to process them
			tweetfile = open(os.path.join(GlobalStore.scriptfolder, 'data', 'tweets', "{}.txt".format(username)), "a")
			for tweet in tweets:
				tweetfile.write(tweet['text'].replace('\n', ' ').encode(encoding='utf-8', errors='replace') + '\n')
			tweetfile.close()
			#Get the id of the last tweet in the list (the newest one), so we know where to start downloading from next time
			storedInfo[username]['highestIdDownloaded'] = tweets[-1]['id']
			storedInfo[username]['linecount'] += len(tweets)

		#Save the stored info to disk too, for future lookups
		with open(twitterInfoFilename, 'w') as twitterFile:
			twitterFile.write(json.dumps(storedInfo))

		self.isUpdating = False
		self.logInfo("[STtip] Updating tweets took {} seconds".format(time.time() - starttime))
<|endoftext|>