<commit_before># -*- coding: utf-8 -*-

"""
    @package
    Generate a HTML BOM list.
    Components are sorted and grouped by value
    Any existing fields are read
"""

from __future__ import print_function
import sys
import os.path
import xml.sax as sax

from bomlib.component import (Component, ComponentGroup)

from bomlib.preferences import BomPref

# -----</Configure>---------------------------------------------------------------

class xmlElement():
    """xml element which can represent all nodes of the netlist tree.  It can be
    used to easily generate various output formats by propogating format
    requests to children recursively.
    """
    def __init__(self, name, parent=None):
        self.name = name
        self.attributes = {}
        self.parent = parent
        self.chars = ""
        self.children = []

    def __str__(self):
        """String representation of this netlist element

        """
        return self.name + "[" + self.chars + "]" + " attr_count:" + str(len(self.attributes))

    def formatXML(self, nestLevel=0, amChild=False):
        """Return this element formatted as XML

        Keywords:
        nestLevel -- increases by one for each level of nesting.
        amChild -- If set to True, the start of document is not returned.

        """
        s = ""
        indent = "    " * nestLevel

        if not amChild:
            s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"

        s += indent + "<" + self.name
        for a in self.attributes:
            s += " " + a + "=\"" + self.attributes[a] + "\""

        if (len(self.chars) == 0) and (len(self.children) == 0):
            s += "/>"
        else:
            s += ">" + self.chars

        for c in self.children:
            s += "\n"
            s += c.formatXML(nestLevel + 1, True)

        if (len(self.children) > 0):
            s += "\n" + indent

        if (len(self.children) > 0) or (len(self.chars) > 0):
            s += "</" + self.name + ">"

        return s

    def formatHTML(self, amChild=False):
        """Return this element formatted as HTML

        Keywords:
        amChild -- If set to True, the start of document is not returned

        """
        s = ""

        if not amChild:
            s = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
                "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

                <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
                <title></title>

                """

        s += "
" + self.name + "
" + self.chars + "
"
        for a in self.attributes:
            s += "
" + a + " = " + self.attributes[a] + "
"

        s += "

\n"

        for c in self.children:
            s += c.formatHTML(True)

        if not amChild:
            s += """

                """

        return s

    def addAttribute(self, attr, value):
        """Add an attribute to this element"""
        self.attributes[attr] = value

    def setAttribute(self, attr, value):
        """Set an attributes value - in fact does the same thing as add
        attribute

        """
        self.attributes[attr] = value

    def setChars(self, chars):
        """Set the characters for this element"""
        self.chars = chars

    def addChars(self, chars):
        """Add characters (textual value) to this element"""
        self.chars += chars

    def addChild(self, child):
        """Add a child element to this element"""
        self.children.append(child)
        return self.children[len(self.children) - 1]

    def getParent(self):
        """Get the parent of this element (Could be None)"""
        return self.parent

    def getChild(self, name):
        """Returns the first child element named 'name'

        Keywords:
        name -- The name of the child element to return"""
        for child in self.children:
            if child.name == name:
                return child
        return None

    def getChildren(self, name=None):
        if name:
            # return _all_ children named "name"
            ret = []
            for child in self.children:
                if child.name == name:
                    ret.append(child)
            return ret
        else:
            return self.children

    def get(self, elemName, attribute="", attrmatch=""):
        """Return the text data for either an attribute or an xmlElement
        """
        if (self.name == elemName):
            if attribute != "":
                try:
                    if attrmatch != "":
                        if self.attributes[attribute] == attrmatch:
                            return self.chars
                    else:
                        return self.attributes[attribute]
                except AttributeError:
                    return ""
            else:
                return self.chars

        for child in self.children:
            ret = child.get(elemName, attribute, attrmatch)
            if ret != "":
                return ret

        return ""

class libpart():
    """Class for a library part, aka 'libpart' in the xml netlist file.
    (Components in eeschema are instantiated from library parts.)
    This part class is implemented by wrapping an xmlElement with accessors.
    This xmlElement instance is held in field 'element'.
    """
    def __init__(self, xml_element):
        #
        self.element = xml_element

    def getLibName(self):
        return self.element.get("libpart", "lib")

    def getPartName(self):
        return self.element.get("libpart", "part")

    # For backwards Compatibility with v4.x only
    def getDescription(self):
        return self.element.get("description")

    def getDocs(self):
        return self.element.get("docs")

    def getField(self, name):
        return self.element.get("field", "name", name)

    def getFieldNames(self):
        """Return a list of field names in play for this libpart.
        """
        fieldNames = []
        fields = self.element.getChild('fields')
        if fields:
            for f in fields.getChildren():
                fieldNames.append(f.get('field', 'name'))
        return fieldNames

    def getDatasheet(self):

        datasheet = self.getField("Datasheet")

        if not datasheet or datasheet == "":
            docs = self.getDocs()

            if "http" in docs or ".pdf" in docs:
                datasheet = docs

        return datasheet

    def getFootprint(self):
        return self.getField("Footprint")

    def getAliases(self):
        """Return a list of aliases or None"""
        aliases = self.element.getChild("aliases")
        if aliases:
            ret = []
            children = aliases.getChildren()
            # grab the text out of each child:
            for child in children:
                ret.append(child.get("alias"))
            return ret
        return None

class netlist():
    """ KiCad generic netlist class. Generally loaded from a KiCad generic
    netlist file. Includes several helper functions to ease BOM creating
    scripts

    """
    def __init__(self, fname="", prefs=None):
        """Initialiser for the genericNetlist class

        Keywords:
        fname -- The name of the generic netlist file to open (Optional)

        """
        self.design = None
        self.components = []
        self.libparts = []
        self.libraries = []
        self.nets = []

        # The entire tree is loaded into self.tree
        self.tree = []

        self._curr_element = None

        if not prefs:
            prefs = BomPref()  # Default values

        self.prefs = prefs

        if fname != "":
            self.load(fname)

    def addChars(self, content):
        """Add characters to the current element"""
        self._curr_element.addChars(content)

    def addElement(self, name):
        """Add a new KiCad generic element to the list"""
        if self._curr_element is None:
            self.tree = xmlElement(name)
            self._curr_element = self.tree
        else:
            self._curr_element = self._curr_element.addChild(
                xmlElement(name, self._curr_element))

        # If this element is a component, add it to the components list
        if self._curr_element.name == "comp":
            self.components.append(Component(self._curr_element, prefs=self.prefs))

        # Assign the design element
        if self._curr_element.name == "design":
            self.design = self._curr_element

        # If this element is a library part, add it to the parts list
        if self._curr_element.name == "libpart":
            self.libparts.append(libpart(self._curr_element))

        # If this element is a net, add it to the nets list
        if self._curr_element.name == "net":
            self.nets.append(self._curr_element)

        # If this element is a library, add it to the libraries list
        if self._curr_element.name == "library":
            self.libraries.append(self._curr_element)

        return self._curr_element

    def endDocument(self):
        """Called when the netlist document has been fully parsed"""
        # When the document is complete, the library parts must be linked to
        # the components as they are seperate in the tree so as not to
        # duplicate library part information for every component
        for c in self.components:
            for p in self.libparts:
                if p.getLibName() == c.getLibName():
                    if p.getPartName() == c.getPartName():
                        c.setLibPart(p)
                        break
                    else:
                        aliases = p.getAliases()
                        if aliases and self.aliasMatch(c.getPartName(), aliases):
                            c.setLibPart(p)
                            break

            if not c.getLibPart():
                print('missing libpart for ref:', c.getRef(), c.getPartName(), c.getLibName())

    def aliasMatch(self, partName, aliasList):
        for alias in aliasList:
            if partName == alias:
                return True
        return False

    def endElement(self):
        """End the current element and switch to its parent"""
        self._curr_element = self._curr_element.getParent()

    def getDate(self):
        """Return the date + time string generated by the tree creation tool"""
        if (sys.version_info[0] >= 3):
            return self.design.get("date")
        else:
            return self.design.get("date").encode('ascii', 'ignore')

    def getSource(self):

        path = self.design.get("source").replace("\\", "/")
        path = os.path.basename(path)

        """Return the source string for the design"""
        if (sys.version_info[0] >= 3):
            return path
        else:
            return path.encode('ascii', 'ignore')

    def getTool(self):
        """Return the tool string which was used to create the netlist tree"""
        if (sys.version_info[0] >= 3):
            return self.design.get("tool")
        else:
            return self.design.get("tool").encode('ascii', 'ignore')

    def getSheet(self):
        return self.design.getChild("sheet")

    def getSheetDate(self):
        sheet = self.getSheet()
        if sheet is None:
            return ""
        return sheet.get("date")

    def getVersion(self):
        """Return the verison of the sheet info"""

        sheet = self.getSheet()

        if sheet is None:
            return ""

        return sheet.get("rev")

    def getInterestingComponents(self):

        # Copy out the components
        ret = [c for c in self.components]

        # Sort first by ref as this makes for easier to read BOM's
        ret.sort(key=lambda g: g.getRef())

        return ret

    def groupComponents(self, components):

        groups = []

        # Iterate through each component, and test whether a group for these already exists
        for c in components:

            if self.prefs.useRegex:
                # Skip components if they do not meet regex requirements
                if not c.testRegInclude():
                    continue
                if c.testRegExclude():
                    continue

            found = False

            for g in groups:
                if g.matchComponent(c):
                    g.addComponent(c)
                    found = True
                    break

            if not found:
                g = ComponentGroup(prefs=self.prefs)  # Pass down the preferences
                g.addComponent(c)
                groups.append(g)

        # Sort the references within each group
        for g in groups:
            g.sortComponents()
            g.updateFields(self.prefs.useAlt, self.prefs.altWrap)

        # Sort the groups
        # First priority is the Type of component (e.g. R?, U?, L?)
        groups = sorted(groups, key=lambda g: [g.components[0].getPrefix(), g.components[0].getValue()])

        return groups

    def formatXML(self):
        """Return the whole netlist formatted in XML"""
        return self.tree.formatXML()

    def formatHTML(self):
        """Return the whole netlist formatted in HTML"""
        return self.tree.formatHTML()

    def load(self, fname):
        """Load a KiCad generic netlist

        Keywords:
        fname -- The name of the generic netlist file to open

        """
        try:
            self._reader = sax.make_parser()
            self._reader.setContentHandler(_gNetReader(self))
            self._reader.parse(fname)
        except IOError as e:
            print(__file__, ":", e, file=sys.stderr)
            sys.exit(-1)

class _gNetReader(sax.handler.ContentHandler):
    """SAX KiCad generic netlist content handler - passes most of the work back
    to the 'netlist' class which builds a complete tree in RAM for the design

    """
    def __init__(self, aParent):
        self.parent = aParent

    def startElement(self, name, attrs):
        """Start of a new XML element event"""
        element = self.parent.addElement(name)

        for name in attrs.getNames():
            element.addAttribute(name, attrs.getValue(name))

    def endElement(self, name):
        self.parent.endElement()

    def characters(self, content):
        # Ignore erroneous white space - ignoreableWhitespace does not get rid
        # of the need for this!
        if not content.isspace():
            self.parent.addChars(content)

    def endDocument(self):
        """End of the XML document event"""
        self.parent.endDocument()
<commit_msg>Remove some unused code<commit_after># -*- coding: utf-8 -*-

"""
    @package
    Generate a HTML BOM list.
    Components are sorted and grouped by value
    Any existing fields are read
"""

from __future__ import print_function
import sys
import os.path
import xml.sax as sax

from bomlib.component import (Component, ComponentGroup)

from bomlib.preferences import BomPref

# -----</Configure>---------------------------------------------------------------

class xmlElement():
    """xml element which can represent all nodes of the netlist tree.  It can be
    used to easily generate various output formats by propogating format
    requests to children recursively.
    """
    def __init__(self, name, parent=None):
        self.name = name
        self.attributes = {}
        self.parent = parent
        self.chars = ""
        self.children = []

    def __str__(self):
        """String representation of this netlist element

        """
        return self.name + "[" + self.chars + "]" + " attr_count:" + str(len(self.attributes))

    def addAttribute(self, attr, value):
        """Add an attribute to this element"""
        self.attributes[attr] = value

    def setAttribute(self, attr, value):
        """Set an attributes value - in fact does the same thing as add
        attribute

        """
        self.attributes[attr] = value

    def setChars(self, chars):
        """Set the characters for this element"""
        self.chars = chars

    def addChars(self, chars):
        """Add characters (textual value) to this element"""
        self.chars += chars

    def addChild(self, child):
        """Add a child element to this element"""
        self.children.append(child)
        return self.children[len(self.children) - 1]

    def getParent(self):
        """Get the parent of this element (Could be None)"""
        return self.parent

    def getChild(self, name):
        """Returns the first child element named 'name'

        Keywords:
        name -- The name of the child element to return"""
        for child in self.children:
            if child.name == name:
                return child
        return None

    def getChildren(self, name=None):
        if name:
            # return _all_ children named "name"
            ret = []
            for child in self.children:
                if child.name == name:
                    ret.append(child)
            return ret
        else:
            return self.children

    def get(self, elemName, attribute="", attrmatch=""):
        """Return the text data for either an attribute or an xmlElement
        """
        if (self.name == elemName):
            if attribute != "":
                try:
                    if attrmatch != "":
                        if self.attributes[attribute] == attrmatch:
                            return self.chars
                    else:
                        return self.attributes[attribute]
                except AttributeError:
                    return ""
            else:
                return self.chars

        for child in self.children:
            ret = child.get(elemName, attribute, attrmatch)
            if ret != "":
                return ret

        return ""

class libpart():
    """Class for a library part, aka 'libpart' in the xml netlist file.
    (Components in eeschema are instantiated from library parts.)
    This part class is implemented by wrapping an xmlElement with accessors.
    This xmlElement instance is held in field 'element'.
    """
    def __init__(self, xml_element):
        #
        self.element = xml_element

    def getLibName(self):
        return self.element.get("libpart", "lib")

    def getPartName(self):
        return self.element.get("libpart", "part")

    # For backwards Compatibility with v4.x only
    def getDescription(self):
        return self.element.get("description")

    def getDocs(self):
        return self.element.get("docs")

    def getField(self, name):
        return self.element.get("field", "name", name)

    def getFieldNames(self):
        """Return a list of field names in play for this libpart.
        """
        fieldNames = []
        fields = self.element.getChild('fields')
        if fields:
            for f in fields.getChildren():
                fieldNames.append(f.get('field', 'name'))
        return fieldNames

    def getDatasheet(self):

        datasheet = self.getField("Datasheet")

        if not datasheet or datasheet == "":
            docs = self.getDocs()

            if "http" in docs or ".pdf" in docs:
                datasheet = docs

        return datasheet

    def getFootprint(self):
        return self.getField("Footprint")

    def getAliases(self):
        """Return a list of aliases or None"""
        aliases = self.element.getChild("aliases")
        if aliases:
            ret = []
            children = aliases.getChildren()
            # grab the text out of each child:
            for child in children:
                ret.append(child.get("alias"))
            return ret
        return None

class netlist():
    """ KiCad generic netlist class. Generally loaded from a KiCad generic
    netlist file. Includes several helper functions to ease BOM creating
    scripts

    """
    def __init__(self, fname="", prefs=None):
        """Initialiser for the genericNetlist class

        Keywords:
        fname -- The name of the generic netlist file to open (Optional)

        """
        self.design = None
        self.components = []
        self.libparts = []
        self.libraries = []
        self.nets = []

        # The entire tree is loaded into self.tree
        self.tree = []

        self._curr_element = None

        if not prefs:
            prefs = BomPref()  # Default values

        self.prefs = prefs

        if fname != "":
            self.load(fname)

    def addChars(self, content):
        """Add characters to the current element"""
        self._curr_element.addChars(content)

    def addElement(self, name):
        """Add a new KiCad generic element to the list"""
        if self._curr_element is None:
            self.tree = xmlElement(name)
            self._curr_element = self.tree
        else:
            self._curr_element = self._curr_element.addChild(
                xmlElement(name, self._curr_element))

        # If this element is a component, add it to the components list
        if self._curr_element.name == "comp":
            self.components.append(Component(self._curr_element, prefs=self.prefs))

        # Assign the design element
        if self._curr_element.name == "design":
            self.design = self._curr_element

        # If this element is a library part, add it to the parts list
        if self._curr_element.name == "libpart":
            self.libparts.append(libpart(self._curr_element))

        # If this element is a net, add it to the nets list
        if self._curr_element.name == "net":
            self.nets.append(self._curr_element)

        # If this element is a library, add it to the libraries list
        if self._curr_element.name == "library":
            self.libraries.append(self._curr_element)

        return self._curr_element

    def endDocument(self):
        """Called when the netlist document has been fully parsed"""
        # When the document is complete, the library parts must be linked to
        # the components as they are seperate in the tree so as not to
        # duplicate library part information for every component
        for c in self.components:
            for p in self.libparts:
                if p.getLibName() == c.getLibName():
                    if p.getPartName() == c.getPartName():
                        c.setLibPart(p)
                        break
                    else:
                        aliases = p.getAliases()
                        if aliases and self.aliasMatch(c.getPartName(), aliases):
                            c.setLibPart(p)
                            break

            if not c.getLibPart():
                print('missing libpart for ref:', c.getRef(), c.getPartName(), c.getLibName())

    def aliasMatch(self, partName, aliasList):
        for alias in aliasList:
            if partName == alias:
                return True
        return False

    def endElement(self):
        """End the current element and switch to its parent"""
        self._curr_element = self._curr_element.getParent()

    def getDate(self):
        """Return the date + time string generated by the tree creation tool"""
        if (sys.version_info[0] >= 3):
            return self.design.get("date")
        else:
            return self.design.get("date").encode('ascii', 'ignore')

    def getSource(self):

        path = self.design.get("source").replace("\\", "/")
        path = os.path.basename(path)

        """Return the source string for the design"""
        if (sys.version_info[0] >= 3):
            return path
        else:
            return path.encode('ascii', 'ignore')

    def getTool(self):
        """Return the tool string which was used to create the netlist tree"""
        if (sys.version_info[0] >= 3):
            return self.design.get("tool")
        else:
            return self.design.get("tool").encode('ascii', 'ignore')

    def getSheet(self):
        return self.design.getChild("sheet")

    def getSheetDate(self):
        sheet = self.getSheet()
        if sheet is None:
            return ""
        return sheet.get("date")

    def getVersion(self):
        """Return the verison of the sheet info"""

        sheet = self.getSheet()

        if sheet is None:
            return ""

        return sheet.get("rev")

    def getInterestingComponents(self):

        # Copy out the components
        ret = [c for c in self.components]

        # Sort first by ref as this makes for easier to read BOM's
        ret.sort(key=lambda g: g.getRef())

        return ret

    def groupComponents(self, components):

        groups = []

        # Iterate through each component, and test whether a group for these already exists
        for c in components:

            if self.prefs.useRegex:
                # Skip components if they do not meet regex requirements
                if not c.testRegInclude():
                    continue
                if c.testRegExclude():
                    continue

            found = False

            for g in groups:
                if g.matchComponent(c):
                    g.addComponent(c)
                    found = True
                    break

            if not found:
                g = ComponentGroup(prefs=self.prefs)  # Pass down the preferences
                g.addComponent(c)
                groups.append(g)

        # Sort the references within each group
        for g in groups:
            g.sortComponents()
            g.updateFields(self.prefs.useAlt, self.prefs.altWrap)

        # Sort the groups
        # First priority is the Type of component (e.g. R?, U?, L?)
        groups = sorted(groups, key=lambda g: [g.components[0].getPrefix(), g.components[0].getValue()])

        return groups

    def load(self, fname):
        """Load a KiCad generic netlist

        Keywords:
        fname -- The name of the generic netlist file to open

        """
        try:
            self._reader = sax.make_parser()
            self._reader.setContentHandler(_gNetReader(self))
            self._reader.parse(fname)
        except IOError as e:
            print(__file__, ":", e, file=sys.stderr)
            sys.exit(-1)

class _gNetReader(sax.handler.ContentHandler):
    """SAX KiCad generic netlist content handler - passes most of the work back
    to the 'netlist' class which builds a complete tree in RAM for the design

    """
    def __init__(self, aParent):
        self.parent = aParent

    def startElement(self, name, attrs):
        """Start of a new XML element event"""
        element = self.parent.addElement(name)

        for name in attrs.getNames():
            element.addAttribute(name, attrs.getValue(name))

    def endElement(self, name):
        self.parent.endElement()

    def characters(self, content):
        # Ignore erroneous white space - ignoreableWhitespace does not get rid
        # of the need for this!
        if not content.isspace():
            self.parent.addChars(content)

    def endDocument(self):
        """End of the XML document event"""
        self.parent.endDocument()
<|endoftext|>


<commit_before>#!/usr/bin/python2.7
# coding=utf-8
import os
import urllib2
import json
import subprocess
import re

def main():
	try:
		videos_file = open("videos.json", "r")
		videos = json.load(videos_file)

		playlists_file = open("playlists.json", "r")
		playlists = json.load(playlists_file)
	except:
		print "Please run getlists.py first."
		print "Exiting...."
		exit()
	finally:
		try:
			videos_file.close()
			playlists_file.close()
		except:
			pass #Fail silently. It's no big deal if we can't close them. I hope.

	try:
		done_file = open("done.json", "r")
		done = json.load(done_file)
	except:
		done = []
	finally:
		try:
			done_file.close()
		except:
			pass #Same here.

	try:
		if not os.access("_videos", os.F_OK):
			os.makedirs("_videos")
		if not os.access("_incomplete", os.F_OK):
			os.makedirs("_incomplete")
		for thisPlaylist in playlists:
			if not os.access(thisPlaylist["name"], os.F_OK):
				os.makedirs(thisPlaylist["name"])

		for thisVideo in videos:
			fileName = thisVideo["title"] + " (~" + thisVideo["uploader"] + ")"
			fileName = fileName.replace("/", "%")
			filePath = "_videos/" + fileName

			#'filter(func, list)' returns the item(s) in 'list' for which func(item) is True
			#'lambda arg: foo' defines an anonymous (unnamed) function that takes 'arg' as an input and returns 'foo'
			#so this lambda returns True or False depending on whether the key "idOrUrl" maps to the value thisVideo["pageUrl"]
			#and this whole filter statement returns a list containing the item in 'playlists' that has the specified "idOrUrl"
			#'[0]' (at the end of the line) selects the first (and only) item in the list
			inPlaylist = filter(lambda pl: pl["idOrUrl"] == thisVideo["playlistIdOrUrl"], playlists)[0]
			#thus, 'inPlaylist' is a dictionary describing the playlist in which 'thisVideo' resides

			#note that 'thisVideo' may be in multiple playlists, in which case it will have multiple entries in 'videos', one for each playlist

			if done.count(thisVideo["pageUrl"]) == 0 and os.access(filePath, os.R_OK) and os.path.getsize(filePath) > 0:
				print ""
				print filePath
				if videoIsComplete(filePath):
					print thisVideo["pageUrl"] + " has already been downloaded."
					done.append(thisVideo["pageUrl"])
				else:
					print thisVideo["pageUrl"] + " did not download completely."
					print "Backing up incomplete video...."
					os.rename(filePath, "_incomplete/" + fileName)
					print "Video will now be re-downloaded."

			if done.count(thisVideo["pageUrl"]) == 0: #thisVideo["pageUrl"] does not appear in 'done.json'
				print ""
				print thisVideo["pageUrl"] + ":"
				if thisVideo["title"] == "":
					print "WARNING: Video has blank title."
					#I'm not sure why we're supporting videos with blank titles. It just seems like the right thing to do.
				print "Downloading to '" + filePath + "'...."

				youtube_dlProcess = subprocess.Popen(['youtube-dl', '-g', thisVideo["pageUrl"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
				youtube_dlProcessOutput = youtube_dlProcess.communicate()
				# 'communicate()' returns (stdout, stderr)
				if youtube_dlProcessOutput[0] == "": #if no output on stdout (i.e., something went wrong)
					if re.search('HTTP Error 402', youtube_dlProcessOutput[1]) != None: #if regex matches
						print ""
						print "FATAL ERROR: YouTube thinks you've used too much bandwidth."
						print "FATAL ERROR: Wait a few minutes and try again."
						raise KeyboardInterrupt
						#Really we should define a custom exception for terminating in a safe manner, but
						#I don't really want to do that right now.
					elif re.search('urlopen error', youtube_dlProcessOutput[1]) != None:
						print ""
						print "FATAL ERROR: Cannot connect to YouTube"
						print "FATAL ERROR: You probably don't have an Internet connection"
						raise KeyboardInterrupt
						#Same as above.
					else:
						print "WARNING: Video was removed from YouTube."
						print "WARNING: youtube-dl returned the following on stderr:"
						print "\t" + youtube_dlProcessOutput[1].rstrip("\n")
						print "WARNING: Skipping video...."
						done.append(thisVideo["pageUrl"])
						#"done" == don't retry later---removed videos don't usually come back.
						#If you want to recheck all videos, delete done.json
						continue #jump to next iteration
				else:
					thisVidRawUrl = youtube_dlProcessOutput[0].rstrip("\n")

					try:
						f = open(filePath, "w")
						payload = urllib2.urlopen(thisVidRawUrl)
						f.write(payload.read())
						if videoIsComplete(filePath):
							done.append(thisVideo["pageUrl"])
							print "Done."
						else:
							print "Video did not download completely."
							print "Backing up incomplete video...."
							os.rename(filePath, "_incomplete/" + fileName)
							print "Video will be re-downloaded next run."
					except urllib2.HTTPError as e:
						if e.code == 404:
							print "WARNING: Video was removed from YouTube."
							print "WARNING: Skipping video...."
							#not fatal!

						elif e.code == 402:
							print "FATAL ERROR: YouTube thinks you've used too much bandwidth."
							print "FATAL ERROR: Wait a few minutes and try again."
							print "FATAL ERROR: Exiting...."
							exit()
						else:
							print "FATAL ERROR: Unknown HTTP Error " + str(e.code)
							exit()
					finally:
						try:
							f.close()
							del f
						except:
							pass #silently fail

			if thisVideo["playlistIndex"] == "":
				symlinkPath = inPlaylist["name"] + "/" + fileName
			else:
				symlinkPath = inPlaylist["name"] + "/" + str(thisVideo["playlistIndex"]).zfill(3) + " " + fileName

			if ( os.access(filePath, os.R_OK) ) and ( not os.access(symlinkPath, os.R_OK) ): #file exists but link does not
				print ""
				symlinkTarget = "../_videos/" + fileName
				print "Symlinking from '" + symlinkPath + "' to '" + symlinkTarget + "'."
				os.symlink(symlinkTarget, symlinkPath)
	except KeyboardInterrupt:
		#In every terminal I've used, ctrl-Cing inserts the text "^C", but no newline.
		#That's why we need two newlines here.
		print ""
		print ""
		print "Terminating...."
	except:
		print ""
		print "FATAL ERROR: Unhandled exception"
		print ""
		raise
	finally:
		try:
			done_file = open("done.json", "w")
			json.dump(done, done_file, separators=(',', ':'), indent=0)
			#`indent=0` inserts newlines but doesn't actually indent.
			done_file.close()
		except:
			print ""
			print "WARNING: Could not write to done.json"
			print "WARNING: Videos you manually removed from _videos might be redownloaded the"
			print "WARNING: next time dlvideos.py runs"
			print ""

def videoIsComplete(filePath):
	ffmpegArgList = ['ffmpeg', '-v', 'debug', '-i', filePath, '-codec', 'copy', '-f', 'rawvideo', '-y', '/dev/null']
	ffmpegProcess = subprocess.Popen(ffmpegArgList, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	ffmpegOutput = ffmpegProcess.communicate()

	#print ffmpegOutput[1]
	linesWithTruncating = re.findall(r'^Truncating.*', ffmpegOutput[1], re.MULTILINE)
	if len(linesWithTruncating) == 0:
		return True
	else:
		for thisLine in linesWithTruncating:
			print "\t" + thisLine
		return False
	return True

if __name__ == "__main__":
	main()
<commit_msg>Fix broken incomplete-file detection<commit_after>#!/usr/bin/python2.7
# coding=utf-8
import os
import urllib2
import json
import subprocess
import re

def main():
	try:
		videos_file = open("videos.json", "r")
		videos = json.load(videos_file)

		playlists_file = open("playlists.json", "r")
		playlists = json.load(playlists_file)
	except:
		print "Please run getlists.py first."
		print "Exiting...."
		exit()
	finally:
		try:
			videos_file.close()
			playlists_file.close()
		except:
			pass #Fail silently. It's no big deal if we can't close them. I hope.

	try:
		done_file = open("done.json", "r")
		done = json.load(done_file)
	except:
		done = []
	finally:
		try:
			done_file.close()
		except:
			pass #Same here.

	try:
		if not os.access("_videos", os.F_OK):
			os.makedirs("_videos")
		if not os.access("_incomplete", os.F_OK):
			os.makedirs("_incomplete")
		for thisPlaylist in playlists:
			if not os.access(thisPlaylist["name"], os.F_OK):
				os.makedirs(thisPlaylist["name"])

		for thisVideo in videos:
			fileName = thisVideo["title"] + " (~" + thisVideo["uploader"] + ")"
			fileName = fileName.replace("/", "%")
			filePath = "_videos/" + fileName

			#'filter(func, list)' returns the item(s) in 'list' for which func(item) is True
			#'lambda arg: foo' defines an anonymous (unnamed) function that takes 'arg' as an input and returns 'foo'
			#so this lambda returns True or False depending on whether the key "idOrUrl" maps to the value thisVideo["pageUrl"]
			#and this whole filter statement returns a list containing the item in 'playlists' that has the specified "idOrUrl"
			#'[0]' (at the end of the line) selects the first (and only) item in the list
			inPlaylist = filter(lambda pl: pl["idOrUrl"] == thisVideo["playlistIdOrUrl"], playlists)[0]
			#thus, 'inPlaylist' is a dictionary describing the playlist in which 'thisVideo' resides

			#note that 'thisVideo' may be in multiple playlists, in which case it will have multiple entries in 'videos', one for each playlist

			if done.count(thisVideo["pageUrl"]) == 0 and os.access(filePath, os.R_OK) and os.path.getsize(filePath) > 0:
				print ""
				print filePath
				if videoIsComplete(filePath):
					print thisVideo["pageUrl"] + " has already been downloaded."
					done.append(thisVideo["pageUrl"])
				else:
					print "WARNING: " + thisVideo["pageUrl"] + " did not download completely."
					#FIXME: should this be a warning or an error?
					print "WARNING: Backing up incomplete video...."
					os.rename(filePath, "_incomplete/" + fileName)
					print "WARNING: Video will now be re-downloaded."

			if done.count(thisVideo["pageUrl"]) == 0: #thisVideo["pageUrl"] does not appear in 'done.json'
				print ""
				print thisVideo["pageUrl"] + ":"
				if thisVideo["title"] == "":
					print "WARNING: Video has blank title."
					#I'm not sure why we're supporting videos with blank titles. It just seems like the right thing to do.

				print "Retrieving raw video URL...."
				youtube_dlProcess = subprocess.Popen(['youtube-dl', '-g', thisVideo["pageUrl"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
				youtube_dlProcessOutput = youtube_dlProcess.communicate()
				# 'communicate()' returns (stdout, stderr)
				if youtube_dlProcessOutput[0] == "": #if no output on stdout (i.e., something went wrong)
					if re.search('HTTP Error 402', youtube_dlProcessOutput[1]) != None: #if regex matches
						print ""
						print "FATAL ERROR: YouTube thinks you've used too much bandwidth."
						print "FATAL ERROR: Wait a few minutes and try again."
						raise KeyboardInterrupt
						#Really we should define a custom exception for terminating in a safe manner, but
						#I don't really want to do that right now.
					elif re.search('urlopen error', youtube_dlProcessOutput[1]) != None:
						print ""
						print "FATAL ERROR: Cannot connect to YouTube"
						print "FATAL ERROR: You probably don't have an Internet connection"
						raise KeyboardInterrupt
						#Same as above.
					else:
						print "WARNING: Video was removed from YouTube."
						print "WARNING: youtube-dl returned the following on stderr:"
						print "\t" + youtube_dlProcessOutput[1].rstrip("\n")
						print "WARNING: Skipping video...."
						done.append(thisVideo["pageUrl"])
						#"done" means don't retry later---removed videos don't usually come back.
						#If you want to recheck all videos, delete done.json
						continue #jump to next iteration
				else:
					thisVidRawUrl = youtube_dlProcessOutput[0].rstrip("\n")

					tryAgain = True
					while tryAgain:
						try:
							print "Downloading to '" + filePath + "'...."
							f = open(filePath, "w")
							payload = urllib2.urlopen(thisVidRawUrl)
							f.write(payload.read())
							if videoIsComplete(filePath):
								done.append(thisVideo["pageUrl"])
								print "Done."
								tryAgain = False
							else:
								print "WARNING: Video did not download completely."
								#FIXME: should this be a warning or an error?
								#print "Backing up incomplete video...."
								#os.rename(filePath, "_incomplete/" + fileName)
								#It turns out that renaming (moving) an open file is a dumb idea.
								print "WARNING: Video will now be re-downloaded."
								#tryAgain remains True
						except urllib2.HTTPError as e:
							if e.code == 404:
								print "WARNING: Video was removed from YouTube."
								print "WARNING: Skipping video...."
								done.append(thisVideo["pageUrl"])
								tryAgain = False
								#not fatal!
							elif e.code == 402:
								print "FATAL ERROR: YouTube thinks you've used too much bandwidth."
								print "FATAL ERROR: Wait a few minutes and try again."
								print "FATAL ERROR: Exiting...."
								exit()
							else:
								print "FATAL ERROR: Unknown HTTP Error " + str(e.code)
								exit()
						finally:
							try:
								f.close()
							except:
								pass #silently fail
							finally:
								del f

			if thisVideo["playlistIndex"] == "":
				symlinkPath = inPlaylist["name"] + "/" + fileName
			else:
				symlinkPath = inPlaylist["name"] + "/" + str(thisVideo["playlistIndex"]).zfill(3) + " " + fileName

			if ( os.access(filePath, os.R_OK) ) and ( not os.access(symlinkPath, os.R_OK) ): #file exists but link does not
				print ""
				symlinkTarget = "../_videos/" + fileName
				print "Symlinking from '" + symlinkPath + "' to '" + symlinkTarget + "'."
				os.symlink(symlinkTarget, symlinkPath)
	except KeyboardInterrupt:
		#In every terminal I've used, ctrl-Cing inserts the text "^C", but no newline.
		#That's why we need two newlines here.
		print ""
		print ""
		print "Terminating...."
	except:
		print ""
		print "FATAL ERROR: Unhandled exception"
		print ""
		raise
	finally:
		try:
			done_file = open("done.json", "w")
			json.dump(done, done_file, separators=(',', ':'), indent=0)
			#`indent=0` inserts newlines but doesn't actually indent.
			done_file.close()
		except:
			print ""
			print "WARNING: Could not write to done.json"
			print "WARNING: Videos you manually removed from _videos might be redownloaded the"
			print "WARNING: next time dlvideos.py runs"
			print ""

def videoIsComplete(filePath):
	ffmpegArgList = ['ffmpeg', '-v', 'debug', '-i', filePath, '-codec', 'copy', '-f', 'rawvideo', '-y', '/dev/null']
	ffmpegProcess = subprocess.Popen(ffmpegArgList, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
	#global ffmpegOutput
	ffmpegOutput = ffmpegProcess.communicate()
	linesWithTruncating = re.findall(r'^Truncating.*', ffmpegOutput[1], re.MULTILINE)
	#exit()
	if len(linesWithTruncating) == 0:
		#print "ffmpeg didn't Truncate! Hooray!"
		return True
	else:
		for thisLine in linesWithTruncating:
			print "\t" + thisLine
		return False
	return True

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import codecs
import datetime
import logging
import re
from time import sleep
from typing import List

from telegram import InlineKeyboardButton, InlineKeyboardMarkup
from telegram.error import BadRequest, TelegramError
from telegram.ext.dispatcher import run_async

import helpers
import mdformat
import settings
import util
from custemoji import Emoji
from dialog import messages
from model import Bot, Country
from model import Category
from model import Statistic
from model import track_activity
from model.channel import Channel
from model import Notifications
from model.revision import Revision
from util import restricted

logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
log = logging.getLogger(__name__)

def _format_category_bots(category):
    cat_bots = Bot.of_category_without_new(category)
    text = '*' + str(category) + '*\n'
    text += '\n'.join([str(b) for b in cat_bots])
    return text

class BotList:
    NEW_BOTS_FILE = 'files/new_bots_list.txt'
    CATEGORY_LIST_FILE = 'files/category_list.txt'
    ENGLISH_INTRO_TEXT = 'files/intro_en.txt'
    SPANISH_INTRO_TEXT = 'files/intro_es.txt'

    def __init__(self, bot, update, channel, resend, silent):
        if not channel:
            self.notify_admin_err(
                "I don't know the channel `{}`. Please make sure I am an admin there, "
                "and send a random message so that I can remember the channel meta data.".format(
                    self.channel.username))
            return

        self.bot = bot
        self.update = update
        self.channel = channel
        self.resend = resend
        self.silent = silent
        self.sent = dict()
        self.sent['category'] = list()
        self.chat_id = update.effective_chat.id
        self.message_id = util.mid_from_update(update)

    def notify_admin(self, txt):
        self.bot.formatter.send_or_edit(self.chat_id, Emoji.HOURGLASS_WITH_FLOWING_SAND + ' ' + txt,
                                        to_edit=self.message_id,
                                        disable_web_page_preview=True, disable_notification=True)

    def notify_admin_err(self, txt):
        self.bot.formatter.send_or_edit(self.chat_id, util.failure(txt),
                                        to_edit=self.message_id,
                                        disable_web_page_preview=True, disable_notification=True)

    def _delete_message(self, message_id):
        self.bot.delete_message(self.channel.chat_id, message_id)

    def _save_channel(self):
        self.channel.save()

    @staticmethod
    def create_hyperlink(message_id):
        return 'https://t.me/{}/{}'.format(settings.SELF_CHANNEL_USERNAME, message_id)

    @property
    def portal_markup(self):
        buttons = [
            InlineKeyboardButton("🔺 1⃣ Categories 📚 🔺",
                                 url=BotList.create_hyperlink(self.channel.category_list_mid)),
            InlineKeyboardButton("▫ 2⃣ BotList Bot 🤖 ▫",
                                 url='https://t.me/botlistbot?start'),
            InlineKeyboardButton("▫ 3⃣ BotList Chat 👥💬 ▫",
                                 url='https://t.me/botlistchat'),
            # InlineKeyboardButton("Add to Group 🤖",
            #                      url='https://t.me/botlistbot?startgroup=start'),
        ]
        return InlineKeyboardMarkup(util.build_menu(buttons, 1))

    @staticmethod
    def _read_file(filename):
        with codecs.open(filename, 'r', 'utf-8') as f:
            return f.read()

    def send_or_edit(self, text, message_id, reply_markup=None):
        sleep(3)
        try:
            if self.resend:
                return util.send_md_message(self.bot, self.channel.chat_id, text, timeout=120,
                                            disable_notification=True, reply_markup=reply_markup)
            else:
                if reply_markup:
                    return self.bot.formatter.send_or_edit(self.channel.chat_id, text,
                                                           to_edit=message_id,
                                                           timeout=120, disable_web_page_preview=True,
                                                           disable_notification=True, reply_markup=reply_markup)
                else:
                    return self.bot.formatter.send_or_edit(self.channel.chat_id, text,
                                                           to_edit=message_id,
                                                           timeout=120, disable_web_page_preview=True,
                                                           disable_notification=True)
        except BadRequest as e:
            print(e)
            if 'chat not found' in e.message.lower():
                self.notify_admin_err(
                    "I can't reach BotList Bot with chat-id `{}` (CHAT NOT FOUND error). "
                    "There's probably something wrong with the database.".format(
                        self.channel.chat_id))
                raise e
            if 'message not modified' in e.message.lower():
                return None
            else:
                log.error(e)
                raise e

    def update_intro(self):
        if self.resend:
            self.notify_admin("Sending intro GIF...")
            self.bot.sendDocument(self.channel.chat_id, open("assets/gif/animation.gif", 'rb'), timeout=120)
            sleep(1)

        intro_en = self._read_file(self.ENGLISH_INTRO_TEXT)
        intro_es = self._read_file(self.SPANISH_INTRO_TEXT)

        self.notify_admin("Sending english channel intro text...")
        msg_en = self.send_or_edit(intro_en, self.channel.intro_en_mid)
        self.notify_admin("Sending spanish channel intro text...")
        msg_es = self.send_or_edit(intro_es, self.channel.intro_es_mid)

        if msg_en:
            self.sent['intro_en'] = "English intro sent"
            self.channel.intro_en_mid = msg_en.message_id
        if msg_es:
            self.channel.intro_es_mid = msg_es.message_id
            self.sent['intro_es'] = "Spanish intro sent"
        self._save_channel()

    def update_new_bots_list(self):
        text = self._read_file(self.NEW_BOTS_FILE)

        # insert spaces and the name of the bot
        new_bots_joined = Bot.get_new_bots_markdown()
        text = text.format(new_bots_joined)

        msg = self.send_or_edit(text, self.channel.new_bots_mid)
        self.sent['new_bots_list'] = "List of new bots sent"
        if msg:
            self.channel.new_bots_mid = msg.message_id
        self._save_channel()

    def update_category_list(self):
        self.notify_admin('Sending category list...')

        # generate category links to previous messages
        all_categories = '\n'.join(["[{}](https://t.me/{}/{})".format(
            str(c),
            self.channel.username,
            c.current_message_id
        ) for c in Category.select_all()])

        url_stub = 'https://t.me/{}/'.format(self.channel.username)
        category_list = self._read_file(self.CATEGORY_LIST_FILE)

        # Insert placeholders
        text = category_list.format(
            url_stub + str(self.channel.intro_en_mid),
            url_stub + str(self.channel.intro_es_mid),
            all_categories,
            url_stub + str(self.channel.new_bots_mid)
        )

        msg = self.send_or_edit(text, self.channel.category_list_mid)

        if msg:
            self.channel.category_list_mid = msg.message_id
            self.sent['category_list'] = "Category Links sent"
        self._save_channel()

    def update_categories(self, categories: List):
        counter = 0
        n = len(categories)
        for cat in categories:
            text = _format_category_bots(cat)

            counter += 1
            if counter % 5 == 1:
                self.notify_admin("Sending/Updating categories *{} to {}* ({} total)...".format(
                    counter, n if counter + 4 > n else counter + 4, n
                ))

            msg = self.send_or_edit(text, cat.current_message_id)
            if msg:
                cat.current_message_id = msg.message_id
                self.sent['category'].append("Updated {}".format(cat))
            cat.save()

        self._save_channel()

        # Add "share", "up", and "down" buttons
        for i in range(0, len(categories)):
            buttons = list()
            if i > 0:
                # Not first category
                # Add "Up" button
                buttons.append(InlineKeyboardButton(
                    "🔺",
                    url=BotList.create_hyperlink(categories[i - 1].current_message_id)))

            buttons.append(
                InlineKeyboardButton("Share", url="https://t.me/{}?start={}".format(
                    settings.SELF_BOT_NAME,
                    categories[i].id)))

            if i < len(categories) - 1:
                # Not last category
                buttons.append(InlineKeyboardButton(
                    "🔻",
                    url=BotList.create_hyperlink(categories[i + 1].current_message_id)))

            reply_markup = InlineKeyboardMarkup([buttons])
            self.bot.edit_message_reply_markup(self.channel.chat_id, categories[i].current_message_id,
                                               reply_markup=reply_markup, timeout=60)

    def send_footer(self):
        num_bots = Bot.select_approved().count()
        self.notify_admin('Sending footer...')

        # add footer as notification
        footer = '\n```'
        footer += '\n' + mdformat.centered(
            "• @BotList •\nRevision {}\n{}\n{} bots".format(
                Revision.get_instance().nr,
                datetime.date.today().strftime("%d-%m-%Y"),
                num_bots
            ))
        footer += '```'

        if self.resend or not self.silent:
            try:
                self._delete_message(self.channel.footer_mid)
            except BadRequest as e:
                print(e)
                pass
            footer_to_edit = None
        else:
            footer_to_edit = self.channel.footer_mid

        footer_msg = self.bot.formatter.send_or_edit(self.channel.chat_id, footer,
                                                     to_edit=footer_to_edit,
                                                     timeout=120,
                                                     disable_notifications=self.silent,
                                                     reply_markup=self.portal_markup)
        if footer_msg:
            self.channel.footer_mid = footer_msg.message_id
            self.sent['footer'] = "Footer sent"
        self._save_channel()

    def finish(self):
        # set last update
        self.channel.last_update = datetime.date.today()
        self._save_channel()

        new_bots = Bot.select_new_bots()
        if not self.silent and len(new_bots) > 0:
            self.notify_admin("Sending notifications to subscribers...")
            subscribers = Notifications.select().where(Notifications.enabled == True)
            notification_count = 0
            for sub in subscribers:
                try:
                    util.send_md_message(self.bot, sub.chat_id, messages.BOTLIST_UPDATE_NOTIFICATION.format(
                        n_bots=len(new_bots),
                        new_bots=Bot.get_new_bots_markdown()))
                    notification_count += 1
                    sub.last_notification = datetime.date.today()
                    sub.save()
                except TelegramError:
                    pass
            self.sent['notifications'] = "Notifications sent to {} users.".format(notification_count)

        changes_made = len(self.sent) > 1 or len(self.sent['category']) > 0
        if changes_made:
            text = util.success('{}{}'.format('BotList updated successfully:\n\n', mdformat.results_list(self.sent)))
        else:
            text = mdformat.none_action("No changes were necessary.")

        log.info(self.sent)
        self.bot.formatter.send_or_edit(self.chat_id, text, to_edit=self.message_id)

@restricted(strict=True)
@run_async
def send_botlist(bot, update, resend=False, silent=False):
    log.info("Re-sending BotList..." if resend else "Updating BotList...")

    channel = helpers.get_channel()
    revision = Revision.get_instance()
    revision.nr += 1
    revision.save()

    all_categories = Category.select_all()

    botlist = BotList(bot, update, channel, resend, silent)
    botlist.update_intro()
    botlist.update_categories(all_categories)
    botlist.update_new_bots_list()
    botlist.update_category_list()
    botlist.send_footer()
    botlist.finish()
    channel.save()
    Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)), Statistic.IMPORTANT)

def new_channel_post(bot, update, photo=None):
    post = update.channel_post
    if post.chat.username != settings.SELF_CHANNEL_USERNAME:
        return
    text = post.text

    channel, created = Channel.get_or_create(chat_id=post.chat_id, username=post.chat.username)
    if created:
        channel.save()

    category_list = '•Share your bots to the @BotListChat using the hashtag #new' in text
    intro = 'Hi! Welcome' in text
    category = text[0] == '•' and not category_list
    new_bots_list = 'NEW→' in text

    # TODO: is this a document?
    if photo:
        pass
    elif category:
        try:
            # get the category meta data
            meta = re.match(r'•(.*?)([A-Z].*):(?:\n(.*):)?', text).groups()
            if len(meta) < 2:
                raise ValueError("Category could not get parsed.")

            emojis = str.strip(meta[0])
            name = str.strip(meta[1])
            extra = str.strip(meta[2]) if meta[2] else None
            try:
                cat = Category.get(name=name)
            except Category.DoesNotExist:
                cat = Category(name=name)
            cat.emojis = emojis
            cat.extra = extra
            cat.save()

            # get the bots in that category
            bots = re.findall(r'^(🆕)?.*(@\w+)( .+)?$', text, re.MULTILINE)
            languages = Country.select().execute()
            for b in bots:
                username = b[1]
                try:
                    new_bot = Bot.by_username(username)
                except Bot.DoesNotExist:
                    new_bot = Bot(username=username)

                    print("New bot created: {}".format(username))

                new_bot.category = cat

                new_bot.inlinequeries = "🔎" in b[2]
                new_bot.official = "🔹" in b[2]

                extra = re.findall(r'(\[.*\])', b[2])
                if extra:
                    new_bot.extra = extra[0]

                # find language
                for lang in languages:
                    if lang.emoji in b[2]:
                        new_bot.country = lang

                if b[0]:
                    new_bot.date_added = datetime.date.today()
                else:
                    new_bot.date_added = datetime.date.today() - datetime.timedelta(days=31)

                new_bot.save()
        except AttributeError:
            log.error("Error parsing the following text:\n" + text)
<commit_msg>Implemented re-sending of full botlist<commit_after># -*- coding: utf-8 -*-
import codecs
import datetime
import logging
import re
from time import sleep
from typing import List

from telegram import InlineKeyboardButton, InlineKeyboardMarkup
from telegram.error import BadRequest, TelegramError
from telegram.ext.dispatcher import run_async

import helpers
import mdformat
import settings
import util
from custemoji import Emoji
from dialog import messages
from model import Bot, Country
from model import Category
from model import Notifications
from model import Statistic
from model.channel import Channel
from model.revision import Revision
from util import restricted

logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
log = logging.getLogger(__name__)

def _format_category_bots(category):
    cat_bots = Bot.of_category_without_new(category)
    text = '*' + str(category) + '*\n'
    text += '\n'.join([str(b) for b in cat_bots])
    return text

class BotList:
    NEW_BOTS_FILE = 'files/new_bots_list.txt'
    CATEGORY_LIST_FILE = 'files/category_list.txt'
    ENGLISH_INTRO_TEXT = 'files/intro_en.txt'
    SPANISH_INTRO_TEXT = 'files/intro_es.txt'

    def __init__(self, bot, update, channel, resend, silent):
        if not channel:
            self.notify_admin_err(
                "I don't know the channel `{}`. Please make sure I am an admin there, "
                "and send a random message so that I can remember the channel meta data.".format(
                    self.channel.username))
            return

        self.bot = bot
        self.update = update
        self.channel = channel
        self.resend = resend
        self.silent = silent
        self.sent = dict()
        self.sent['category'] = list()
        self.chat_id = update.effective_chat.id
        self.message_id = util.mid_from_update(update)

    def notify_admin(self, txt):
        self.bot.formatter.send_or_edit(self.chat_id, Emoji.HOURGLASS_WITH_FLOWING_SAND + ' ' + txt,
                                        to_edit=self.message_id,
                                        disable_web_page_preview=True, disable_notification=True)

    def notify_admin_err(self, txt):
        self.bot.formatter.send_or_edit(self.chat_id, util.failure(txt),
                                        to_edit=self.message_id,
                                        disable_web_page_preview=True, disable_notification=True)

    def _delete_message(self, message_id):
        self.bot.delete_message(self.channel.chat_id, message_id)

    def _save_channel(self):
        self.channel.save()

    @staticmethod
    def create_hyperlink(message_id):
        return 'https://t.me/{}/{}'.format(settings.SELF_CHANNEL_USERNAME, message_id)

    @property
    def portal_markup(self):
        buttons = [
            InlineKeyboardButton("🔺 1⃣ Categories 📚 🔺",
                                 url=BotList.create_hyperlink(self.channel.category_list_mid)),
            InlineKeyboardButton("▫ 2⃣ BotList Bot 🤖 ▫",
                                 url='https://t.me/botlistbot?start'),
            InlineKeyboardButton("▫ 3⃣ BotList Chat 👥💬 ▫",
                                 url='https://t.me/botlistchat'),
            # InlineKeyboardButton("Add to Group 🤖",
            #                      url='https://t.me/botlistbot?startgroup=start'),
        ]
        return InlineKeyboardMarkup(util.build_menu(buttons, 1))

    @staticmethod
    def _read_file(filename):
        with codecs.open(filename, 'r', 'utf-8') as f:
            return f.read()

    def send_or_edit(self, text, message_id, reply_markup=None):
        sleep(3)
        try:
            if self.resend:
                return util.send_md_message(self.bot, self.channel.chat_id, text, timeout=120,
                                            disable_notification=True, reply_markup=reply_markup)
            else:
                if reply_markup:
                    return self.bot.formatter.send_or_edit(self.channel.chat_id, text,
                                                           to_edit=message_id,
                                                           timeout=120, disable_web_page_preview=True,
                                                           disable_notification=True, reply_markup=reply_markup)
                else:
                    return self.bot.formatter.send_or_edit(self.channel.chat_id, text,
                                                           to_edit=message_id,
                                                           timeout=120, disable_web_page_preview=True,
                                                           disable_notification=True)
        except BadRequest as e:
            if 'chat not found' in e.message.lower():
                self.notify_admin_err(
                    "I can't reach BotList Bot with chat-id `{}` (CHAT NOT FOUND error). "
                    "There's probably something wrong with the database.".format(
                        self.channel.chat_id))
                raise e
            if 'message not modified' in e.message.lower():
                return None
            else:
                log.error(e)
                raise e

    def update_intro(self):
        if self.resend:
            self.notify_admin("Sending intro GIF...")
            self.bot.sendDocument(self.channel.chat_id, open("assets/gif/animation.gif", 'rb'), timeout=120)
            sleep(1)

        intro_en = self._read_file(self.ENGLISH_INTRO_TEXT)
        intro_es = self._read_file(self.SPANISH_INTRO_TEXT)

        self.notify_admin("Sending english channel intro text...")
        msg_en = self.send_or_edit(intro_en, self.channel.intro_en_mid)
        self.notify_admin("Sending spanish channel intro text...")
        msg_es = self.send_or_edit(intro_es, self.channel.intro_es_mid)

        if msg_en:
            self.sent['intro_en'] = "English intro sent"
            self.channel.intro_en_mid = msg_en.message_id
        if msg_es:
            self.channel.intro_es_mid = msg_es.message_id
            self.sent['intro_es'] = "Spanish intro sent"
        self._save_channel()

    def update_new_bots_list(self):
        text = self._read_file(self.NEW_BOTS_FILE)

        # insert spaces and the name of the bot
        new_bots_joined = Bot.get_new_bots_markdown()
        text = text.format(new_bots_joined)

        msg = self.send_or_edit(text, self.channel.new_bots_mid)
        self.sent['new_bots_list'] = "List of new bots sent"
        if msg:
            self.channel.new_bots_mid = msg.message_id
        self._save_channel()

    def update_category_list(self):
        self.notify_admin('Sending category list...')

        # generate category links to previous messages
        all_categories = '\n'.join(["[{}](https://t.me/{}/{})".format(
            str(c),
            self.channel.username,
            c.current_message_id
        ) for c in Category.select_all()])

        url_stub = 'https://t.me/{}/'.format(self.channel.username)
        category_list = self._read_file(self.CATEGORY_LIST_FILE)

        # Insert placeholders
        text = category_list.format(
            url_stub + str(self.channel.intro_en_mid),
            url_stub + str(self.channel.intro_es_mid),
            all_categories,
            url_stub + str(self.channel.new_bots_mid)
        )

        msg = self.send_or_edit(text, self.channel.category_list_mid)

        if msg:
            self.channel.category_list_mid = msg.message_id
            self.sent['category_list'] = "Category Links sent"
        self._save_channel()

    def update_categories(self, categories: List):
        self.notify_admin("Updating BotList categories to Revision {}...".format(Revision.get_instance().nr))

        for cat in categories:
            text = _format_category_bots(cat)

            msg = self.send_or_edit(text, cat.current_message_id)
            if msg:
                cat.current_message_id = msg.message_id
                self.sent['category'].append("{} {}".format(
                    'Resent' if self.resend else 'Updated',
                    cat
                ))
            cat.save()

        self._save_channel()

        # Add "share", "up", and "down" buttons
        for i in range(0, len(categories)):
            buttons = list()
            if i > 0:
                # Not first category
                # Add "Up" button
                buttons.append(InlineKeyboardButton(
                    "🔺",
                    url=BotList.create_hyperlink(categories[i - 1].current_message_id)))

            buttons.append(
                InlineKeyboardButton("Share", url="https://t.me/{}?start={}".format(
                    settings.SELF_BOT_NAME,
                    categories[i].id)))

            if i < len(categories) - 1:
                # Not last category
                buttons.append(InlineKeyboardButton(
                    "🔻",
                    url=BotList.create_hyperlink(categories[i + 1].current_message_id)))

            reply_markup = InlineKeyboardMarkup([buttons])
            self.bot.edit_message_reply_markup(self.channel.chat_id, categories[i].current_message_id,
                                               reply_markup=reply_markup, timeout=60)

    def send_footer(self):
        num_bots = Bot.select_approved().count()
        self.notify_admin('Sending footer...')

        # add footer as notification
        footer = '\n```'
        footer += '\n' + mdformat.centered(
            "• @BotList •\n{}\n{} bots".format(
                datetime.date.today().strftime("%d-%m-%Y"),
                num_bots
            ))
        footer += '```'

        if self.resend or not self.silent:
            try:
                self._delete_message(self.channel.footer_mid)
            except BadRequest as e:
                pass
            footer_to_edit = None
        else:
            footer_to_edit = self.channel.footer_mid

        footer_msg = self.bot.formatter.send_or_edit(self.channel.chat_id, footer,
                                                     to_edit=footer_to_edit,
                                                     timeout=120,
                                                     disable_notifications=self.silent,
                                                     reply_markup=self.portal_markup)
        if footer_msg:
            self.channel.footer_mid = footer_msg.message_id
            self.sent['footer'] = "Footer sent"
        self._save_channel()

    def finish(self):
        # set last update
        self.channel.last_update = datetime.date.today()
        self._save_channel()

        new_bots = Bot.select_new_bots()
        if not self.silent and len(new_bots) > 0:
            self.notify_admin("Sending notifications to subscribers...")
            subscribers = Notifications.select().where(Notifications.enabled == True)
            notification_count = 0
            for sub in subscribers:
                try:
                    util.send_md_message(self.bot, sub.chat_id, messages.BOTLIST_UPDATE_NOTIFICATION.format(
                        n_bots=len(new_bots),
                        new_bots=Bot.get_new_bots_markdown()))
                    notification_count += 1
                    sub.last_notification = datetime.date.today()
                    sub.save()
                except TelegramError:
                    pass
            self.sent['notifications'] = "Notifications sent to {} users.".format(notification_count)

        changes_made = len(self.sent) > 1 or len(self.sent['category']) > 0
        if changes_made:
            text = util.success('{}{}'.format('BotList updated successfully:\n\n', mdformat.results_list(self.sent)))
        else:
            text = mdformat.none_action("No changes were necessary.")

        log.info(self.sent)
        self.bot.formatter.send_or_edit(self.chat_id, text, to_edit=self.message_id)

    def delete_full_botlist(self):
        all_cats = Category.select_all()
        start = all_cats[0].current_message_id - 3  # Some wiggle room and GIF
        end = all_cats[-1].current_message_id + 4  # Some wiggle room
        self.notify_admin("Deleting all messages...")
        for m in range(start, end):
            try:
                self.bot.delete_message(self.channel.chat_id, m)
            except BadRequest as e:
                pass

@restricted(strict=True)
@run_async
def send_botlist(bot, update, resend=False, silent=False):
    log.info("Re-sending BotList..." if resend else "Updating BotList...")

    channel = helpers.get_channel()
    revision = Revision.get_instance()
    revision.nr += 1
    revision.save()

    all_categories = Category.select_all()

    botlist = BotList(bot, update, channel, resend, silent)
    if resend:
        botlist.delete_full_botlist()
    botlist.update_intro()
    botlist.update_categories(all_categories)
    botlist.update_new_bots_list()
    botlist.update_category_list()
    botlist.send_footer()
    botlist.finish()
    channel.save()
    Statistic.of(update, 'send', 'botlist (resend: {})'.format(str(resend)), Statistic.IMPORTANT)

def new_channel_post(bot, update, photo=None):
    post = update.channel_post
    if post.chat.username != settings.SELF_CHANNEL_USERNAME:
        return
    text = post.text

    channel, created = Channel.get_or_create(chat_id=post.chat_id, username=post.chat.username)
    if created:
        channel.save()

    category_list = '•Share your bots to the @BotListChat using the hashtag #new' in text
    intro = 'Hi! Welcome' in text
    category = text[0] == '•' and not category_list
    new_bots_list = 'NEW→' in text

    # TODO: is this a document?
    if photo:
        pass
    elif category:
        try:
            # get the category meta data
            meta = re.match(r'•(.*?)([A-Z].*):(?:\n(.*):)?', text).groups()
            if len(meta) < 2:
                raise ValueError("Category could not get parsed.")

            emojis = str.strip(meta[0])
            name = str.strip(meta[1])
            extra = str.strip(meta[2]) if meta[2] else None
            try:
                cat = Category.get(name=name)
            except Category.DoesNotExist:
                cat = Category(name=name)
            cat.emojis = emojis
            cat.extra = extra
            cat.save()

            # get the bots in that category
            bots = re.findall(r'^(🆕)?.*(@\w+)( .+)?$', text, re.MULTILINE)
            languages = Country.select().execute()
            for b in bots:
                username = b[1]
                try:
                    new_bot = Bot.by_username(username)
                except Bot.DoesNotExist:
                    new_bot = Bot(username=username)

                new_bot.category = cat

                new_bot.inlinequeries = "🔎" in b[2]
                new_bot.official = "🔹" in b[2]

                extra = re.findall(r'(\[.*\])', b[2])
                if extra:
                    new_bot.extra = extra[0]

                # find language
                for lang in languages:
                    if lang.emoji in b[2]:
                        new_bot.country = lang

                if b[0]:
                    new_bot.date_added = datetime.date.today()
                else:
                    new_bot.date_added = datetime.date.today() - datetime.timedelta(days=31)

                new_bot.save()
        except AttributeError:
            log.error("Error parsing the following text:\n" + text)
<|endoftext|>


<commit_before>"""(c) All rights reserved. ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland, VPSI, 2017"""
from datetime import datetime

from parser.box import Box
from parser.navigation_page import NavigationPage
from settings import JAHIA_DATE_FORMAT
from parser.sidebar import Sidebar
import logging
import re

class PageContent:
    """
    The language specific data of a Page
    """
    def __init__(self, page, language, element):
        self.element = element
        self.page = page
        self.site = page.site
        self.wp_id = None
        self.language = language
        # the relative path, e.g. /team.html
        self.path = ""
        self.vanity_urls = []
        self.title = element.getAttribute("jahia:title")
        self.boxes = []
        self.sidebar = Sidebar()
        self.last_update = ""
        # a list of NavigationPages
        self.navigation = []

        # last update
        self.parse_last_update()

        # sidebar
        self.parse_sidebar()

        # path
        self.set_path()

        # navigation
        self.parse_navigation()

        # add to the site PageContents
        self.site.pages_content_by_path[self.path] = self

    def parse_last_update(self):
        """Parse the last update information"""
        date = self.element.getAttribute("jcr:lastModified")

        try:
            if date:
                self.last_update = datetime.strptime(date, JAHIA_DATE_FORMAT)
            else:
                logging.warning(
                    "%s - parse - For the page id=%s the last update date is empty",
                    self.site.name, self.page.pid)
        except ValueError as e:
            logging.error(
                "%s - parse - Invalid last update date for page %s : '%s'",
                self.site.name, self.page.pid, date)
            raise e

    def parse_sidebar(self):
        """ Parse sidebar """

        # search the sidebar in the page xml content
        children = self.element.childNodes
        for child in children:
            if child.nodeName == "extraList":
                for extra in child.childNodes:
                    if extra.ELEMENT_NODE != extra.nodeType:
                        continue
                    box = Box(site=self.site, page_content=self, element=extra, multibox=True)
                    self.sidebar.boxes.append(box)

        nb_boxes = len(self.sidebar.boxes)

        # if we don't have boxes in this sidebar we check the parents
        if nb_boxes == 0:
            parent = self.page.parent

            while parent:
                sidebar = parent.contents[self.language].sidebar

                # we found a sidebar with boxes, we stop
                if len(sidebar.boxes) > 0:
                    self.sidebar = sidebar
                    break

                # otherwise we continue in the hierarchy
                parent = parent.parent

    def set_path(self):
        """
        Set the page path
        """

        if self.page.is_homepage():
            if "en" == self.language:
                self.vanity_urls = ["/index.html"]
            else:
                self.vanity_urls = ["/index-{}.html".format(self.language)]
        else:
            # Vanity URL can have the following content :
            # one URL ==> '/sciences_donnees$$$true$$$true==='
            # many URLs ==> '/sciences_donnees$$$true$$$true===/sciencesdonnees$$$true$$$false==='
            # many URLs ==> '/sciences_donnees$$$true$$$false===/sciencesdonnees$$$true$$$false==='
            vanity_url = self.element.getAttribute("jahia:urlMappings")
            if vanity_url:
                # Going through exploded parts
                for url in vanity_url.split('$$$'):
                    # Cleaning content
                    url = re.sub(r'(true|false)(===)?', '', url)
                    if url:
                        self.vanity_urls.append(url)
            else:
                # use the old Jahia page id
                self.vanity_urls = ["/page-{}-{}.html".format(self.page.pid, self.language)]

        # FIXME, the prefixing part should be done in exporter
        # add the site root_path at the beginning
        self.path = self.site.root_path + self.vanity_urls[0]

    def parse_navigation(self):
        """Parse the navigation"""

        navigation_pages = self.element.getElementsByTagName("navigationPage")

        for navigation_page in navigation_pages:
            # check if the <navigationPage> belongs to this page
            if not self.site.belongs_to(element=navigation_page, page=self.page):
                continue

            for child in navigation_page.childNodes:
                # internal page declared with <jahia:page>
                if child.nodeName == "jahia:page":
                    template = child.getAttribute("jahia:template")

                    # we don't want the sitemap
                    if not template == "sitemap":
                        ref = child.getAttribute("jcr:uuid")
                        title = child.getAttribute("jahia:title")

                        self.add_navigation_page(type="internal", ref=ref, title=title)

                # internal page declared with <jahia:link>
                elif child.nodeName == "jahia:link":
                    ref = child.getAttribute("jahia:reference")
                    title = child.getAttribute("jahia:title")

                    self.add_navigation_page(type="internal", ref=ref, title=title)

                # external page
                elif child.nodeName == "jahia:url":
                    ref = child.getAttribute("jahia:value")
                    title = child.getAttribute("jahia:title")

                    self.add_navigation_page(type="external", ref=ref, title=title)

    def add_navigation_page(self, type, ref, title):
        """Add a NavigationPage with the given info"""

        navigation_page = NavigationPage(parent=self, type=type, ref=ref, title=title)

        self.navigation.append(navigation_page)
<commit_msg>handle multibox more properly<commit_after>"""(c) All rights reserved. ECOLE POLYTECHNIQUE FEDERALE DE LAUSANNE, Switzerland, VPSI, 2017"""
from datetime import datetime

from parser.box import Box
from parser.navigation_page import NavigationPage
from settings import JAHIA_DATE_FORMAT
from parser.sidebar import Sidebar
import logging
import re

class PageContent:
    """
    The language specific data of a Page
    """
    def __init__(self, page, language, element):
        self.element = element
        self.page = page
        self.site = page.site
        self.wp_id = None
        self.language = language
        # the relative path, e.g. /team.html
        self.path = ""
        self.vanity_urls = []
        self.title = element.getAttribute("jahia:title")
        self.boxes = []
        self.sidebar = Sidebar()
        self.last_update = ""
        # a list of NavigationPages
        self.navigation = []

        # last update
        self.parse_last_update()

        # sidebar
        self.parse_sidebar()

        # path
        self.set_path()

        # navigation
        self.parse_navigation()

        # add to the site PageContents
        self.site.pages_content_by_path[self.path] = self

    def parse_last_update(self):
        """Parse the last update information"""
        date = self.element.getAttribute("jcr:lastModified")

        try:
            if date:
                self.last_update = datetime.strptime(date, JAHIA_DATE_FORMAT)
            else:
                logging.warning(
                    "%s - parse - For the page id=%s the last update date is empty",
                    self.site.name, self.page.pid)
        except ValueError as e:
            logging.error(
                "%s - parse - Invalid last update date for page %s : '%s'",
                self.site.name, self.page.pid, date)
            raise e

    def parse_sidebar(self):
        """ Parse sidebar """

        # search the sidebar in the page xml content
        children = self.element.childNodes
        for child in children:
            if child.nodeName == "extraList":
                for extra in child.childNodes:
                    if extra.ELEMENT_NODE != extra.nodeType:
                        continue
                    multibox = extra.getElementsByTagName("text").length > 1
                    box = Box(site=self.site, page_content=self, element=extra, multibox=multibox)
                    self.sidebar.boxes.append(box)

        nb_boxes = len(self.sidebar.boxes)

        # if we don't have boxes in this sidebar we check the parents
        if nb_boxes == 0:
            parent = self.page.parent

            while parent:
                sidebar = parent.contents[self.language].sidebar

                # we found a sidebar with boxes, we stop
                if len(sidebar.boxes) > 0:
                    self.sidebar = sidebar
                    break

                # otherwise we continue in the hierarchy
                parent = parent.parent

    def set_path(self):
        """
        Set the page path
        """

        if self.page.is_homepage():
            if "en" == self.language:
                self.vanity_urls = ["/index.html"]
            else:
                self.vanity_urls = ["/index-{}.html".format(self.language)]
        else:
            # Vanity URL can have the following content :
            # one URL ==> '/sciences_donnees$$$true$$$true==='
            # many URLs ==> '/sciences_donnees$$$true$$$true===/sciencesdonnees$$$true$$$false==='
            # many URLs ==> '/sciences_donnees$$$true$$$false===/sciencesdonnees$$$true$$$false==='
            vanity_url = self.element.getAttribute("jahia:urlMappings")
            if vanity_url:
                # Going through exploded parts
                for url in vanity_url.split('$$$'):
                    # Cleaning content
                    url = re.sub(r'(true|false)(===)?', '', url)
                    if url:
                        self.vanity_urls.append(url)
            else:
                # use the old Jahia page id
                self.vanity_urls = ["/page-{}-{}.html".format(self.page.pid, self.language)]

        # FIXME, the prefixing part should be done in exporter
        # add the site root_path at the beginning
        self.path = self.site.root_path + self.vanity_urls[0]

    def parse_navigation(self):
        """Parse the navigation"""

        navigation_pages = self.element.getElementsByTagName("navigationPage")

        for navigation_page in navigation_pages:
            # check if the <navigationPage> belongs to this page
            if not self.site.belongs_to(element=navigation_page, page=self.page):
                continue

            for child in navigation_page.childNodes:
                # internal page declared with <jahia:page>
                if child.nodeName == "jahia:page":
                    template = child.getAttribute("jahia:template")

                    # we don't want the sitemap
                    if not template == "sitemap":
                        ref = child.getAttribute("jcr:uuid")
                        title = child.getAttribute("jahia:title")

                        self.add_navigation_page(type="internal", ref=ref, title=title)

                # internal page declared with <jahia:link>
                elif child.nodeName == "jahia:link":
                    ref = child.getAttribute("jahia:reference")
                    title = child.getAttribute("jahia:title")

                    self.add_navigation_page(type="internal", ref=ref, title=title)

                # external page
                elif child.nodeName == "jahia:url":
                    ref = child.getAttribute("jahia:value")
                    title = child.getAttribute("jahia:title")

                    self.add_navigation_page(type="external", ref=ref, title=title)

    def add_navigation_page(self, type, ref, title):
        """Add a NavigationPage with the given info"""

        navigation_page = NavigationPage(parent=self, type=type, ref=ref, title=title)

        self.navigation.append(navigation_page)
<|endoftext|>