<commit_before># @Chr1st-oo

import json
import cfscrape
import requests
import sys
import re
from bs4 import BeautifulSoup
from datetime import date

class RCOUpdater():
    def __init__ (self, link=None, name=None):
        self.BASE = "https://readcomiconline.to/Comic/"
        self.link = link

        if name:
            self.link = self.BASE + self.nameLink(str(name).strip())

        self.session = requests.session()
        self.scraper = cfscrape.create_scraper(sess=self.session)

        self.data = ""
        try:
            self.data = json.load(open("rco-data.json", "r"))
        except Exception as e:
            print("An error occurred : {}".format(repr(e)))
            print("Download the data from {}".format("https://drive.google.com/open?id=1eOjwOQx_LHericcowyBRNIJtZZGMKlp6"))
            print("And paste it inside the comic-dl/comic_dl directory")
            sys.exit()

        if self.link:
            try:
                soup = BeautifulSoup(
                    self.scraper.get(self.link).content,
                    "html.parser"
                )

                divs = soup.find_all("div", {"class": "content space-top"})

                #TOP
                name = divs[0].find("div", {"class": "content_top red"}).text.strip()
                infos = divs[0].find("div", {"class": "col info"}).find_all("p")

                #BOTTOM
                issues = divs[1].find("ul", {"class": "list"}).find_all("li")

                print("You want to add {} in the database. Checking if it already exist.".format(name))
                cid = int(self.alreadyExists(name))

                summary = "N/A"

                try:
                    summaryBox = divs[0].find_all("div", {"class" : "section group"})[1]
                    summary = summaryBox.text.strip()
                except:
                    pass

                if cid:
                    print("{} already exists in the database. Trying to update its details...".format(name))
                    self.data["comics"][cid - 1]["name"] = name
                    self.data["comics"][cid - 1]["genres"] = self.getGenre(infos[0])
                    self.data["comics"][cid - 1]["numOfChapters"] = self.getNumOfChapters(issues)
                    self.data["comics"][cid - 1]["publisher"] = self.getPublisher(infos[1])
                    self.data["comics"][cid - 1]["writer"] = self.getWriter(infos[2])
                    self.data["comics"][cid - 1]["artist"] = self.getArtist(infos[3])
                    self.data["comics"][cid - 1]["publicationDate"] = self.getPublicationDate(infos[4]).strip()
                    self.data["comics"][cid - 1]["status"] = self.getStatus(infos[5]).strip()
                    self.data["comics"][cid - 1]["summary"] = summary
                    self.data["comics"][cid - 1]["link"] = self.link
                else:
                    cid = int(self.getLastId(name)) + 1
                    print("{} does not exist in the database. Adding it to the database...".format(name))
                    newComic = {
                        "no": cid,
                        "name": name,
                        "genres": self.getGenre(infos[0]),
                        "numOfChapters": self.getNumOfChapters(issues),
                        "publisher": self.getPublisher(infos[1]),
                        "writer": self.getWriter(infos[2]),
                        "artist": self.getArtist(infos[3]),
                        "publicationDate": self.getPublicationDate(infos[4]).strip(),
                        "status": self.getStatus(infos[5]).strip(),
                        "summary": summary,
                        "link": self.link
                    }

                    self.data["comics"].append(newComic)
                    self.data["last"] = {
                        "no": self.data["comics"][-1]["no"], "name": self.data["comics"][-1]["name"], "dateAdded": date.today().strftime("%d/%m/%Y")
                    }

                with open("rco-data.json", "w") as out:
                    json.dump(self.data, out, indent=2)

                print("Database updated successfully...")
            except Exception as e:
                print("The link is not available, perhaps you provided an invalid link or name.")
                sys.exit()

    def getLastId(self, name):
        return self.data["last"]["no"]

    def nameLink(self, name):
        return re.sub(r"[^A-Z|^a-z|\-|^0-9]", "", name.replace(" ", "-"))

    def alreadyExists(self, name):
        for datum in self.data["comics"]:
            if name == datum["name"]:
                return datum["no"]

        return False

    def getGenre(self, el):
        try:
            genres = []

            g = el.find_all("a")

            for genre in g:
                genres.append(genre.text)

            return genres
        except:
            return ["N/A"]

    def getPublisher(self, el):
        try:
            publishers = []

            p = el.find_all("a")

            for publisher in p:
                publishers.append(publisher.text)

            return publishers
        except:
            return ["N/A"]

    def getWriter(self, el):
        try:
            writers = []

            w = el.find_all("a")

            for writer in w:
                writers.append(writer.text)

            return writers
        except:
            return ["N/A"]

    def getArtist(self, el):
        try:
            artists = []

            a = el.find_all("a")

            for artist in a:
                artists.append(artist.text)

            return artists
        except:
            return ["N/A"]

    def getPublicationDate(self, el):
        try:
            return el.text.replace("Publication date:", "")
        except:
            "N/A"

    def getStatus(self, el):
        try:
            return el.text.replace("Status:", "")
        except:
            return "N/A"

    def getNumOfChapters(self, el):
        try:
            return len(el)
        except:
            return "N/A"<commit_msg>Update dataUpdate.py<commit_after># @Chr1st-oo

import json
import cloudscraper
import requests
import sys
import re
from bs4 import BeautifulSoup
from datetime import date

class RCOUpdater():
    def __init__ (self, link=None, name=None):
        self.BASE = "https://readcomiconline.to/Comic/"
        self.link = link

        if name:
            self.link = self.BASE + self.nameLink(str(name).strip())

        self.session = requests.session()
        self.scraper = cloudscraper.create_scraper(sess=self.session)

        self.data = ""
        try:
            self.data = json.load(open("rco-data.json", "r"))
        except Exception as e:
            print("An error occurred : {}".format(repr(e)))
            print("Download the data from {}".format("https://drive.google.com/open?id=1eOjwOQx_LHericcowyBRNIJtZZGMKlp6"))
            print("And paste it inside the comic-dl/comic_dl directory")
            sys.exit()

        if self.link:
            try:
                soup = BeautifulSoup(
                    self.scraper.get(self.link).content,
                    "html.parser"
                )

                divs = soup.find_all("div", {"class": "content space-top"})

                #TOP
                name = divs[0].find("div", {"class": "content_top red"}).text.strip()
                infos = divs[0].find("div", {"class": "col info"}).find_all("p")

                #BOTTOM
                issues = divs[1].find("ul", {"class": "list"}).find_all("li")

                print("You want to add {} in the database. Checking if it already exist.".format(name))
                cid = int(self.alreadyExists(name))

                summary = "N/A"

                try:
                    summaryBox = divs[0].find_all("div", {"class" : "section group"})[1]
                    summary = summaryBox.text.strip()
                except:
                    pass

                if cid:
                    print("{} already exists in the database. Trying to update its details...".format(name))
                    self.data["comics"][cid - 1]["name"] = name
                    self.data["comics"][cid - 1]["genres"] = self.getGenre(infos[0])
                    self.data["comics"][cid - 1]["numOfChapters"] = self.getNumOfChapters(issues)
                    self.data["comics"][cid - 1]["publisher"] = self.getPublisher(infos[1])
                    self.data["comics"][cid - 1]["writer"] = self.getWriter(infos[2])
                    self.data["comics"][cid - 1]["artist"] = self.getArtist(infos[3])
                    self.data["comics"][cid - 1]["publicationDate"] = self.getPublicationDate(infos[4]).strip()
                    self.data["comics"][cid - 1]["status"] = self.getStatus(infos[5]).strip()
                    self.data["comics"][cid - 1]["summary"] = summary
                    self.data["comics"][cid - 1]["link"] = self.link
                else:
                    cid = int(self.getLastId(name)) + 1
                    print("{} does not exist in the database. Adding it to the database...".format(name))
                    newComic = {
                        "no": cid,
                        "name": name,
                        "genres": self.getGenre(infos[0]),
                        "numOfChapters": self.getNumOfChapters(issues),
                        "publisher": self.getPublisher(infos[1]),
                        "writer": self.getWriter(infos[2]),
                        "artist": self.getArtist(infos[3]),
                        "publicationDate": self.getPublicationDate(infos[4]).strip(),
                        "status": self.getStatus(infos[5]).strip(),
                        "summary": summary,
                        "link": self.link
                    }

                    self.data["comics"].append(newComic)
                    self.data["last"] = {
                        "no": self.data["comics"][-1]["no"], "name": self.data["comics"][-1]["name"], "dateAdded": date.today().strftime("%d/%m/%Y")
                    }

                with open("rco-data.json", "w") as out:
                    json.dump(self.data, out, indent=2)

                print("Database updated successfully...")
            except Exception as e:
                print("The link is not available, perhaps you provided an invalid link or name.")
                sys.exit()

    def getLastId(self, name):
        return self.data["last"]["no"]

    def nameLink(self, name):
        return re.sub(r"[^A-Z|^a-z|\-|^0-9]", "", name.replace(" ", "-"))

    def alreadyExists(self, name):
        for datum in self.data["comics"]:
            if name == datum["name"]:
                return datum["no"]

        return False

    def getGenre(self, el):
        try:
            genres = []

            g = el.find_all("a")

            for genre in g:
                genres.append(genre.text)

            return genres
        except:
            return ["N/A"]

    def getPublisher(self, el):
        try:
            publishers = []

            p = el.find_all("a")

            for publisher in p:
                publishers.append(publisher.text)

            return publishers
        except:
            return ["N/A"]

    def getWriter(self, el):
        try:
            writers = []

            w = el.find_all("a")

            for writer in w:
                writers.append(writer.text)

            return writers
        except:
            return ["N/A"]

    def getArtist(self, el):
        try:
            artists = []

            a = el.find_all("a")

            for artist in a:
                artists.append(artist.text)

            return artists
        except:
            return ["N/A"]

    def getPublicationDate(self, el):
        try:
            return el.text.replace("Publication date:", "")
        except:
            "N/A"

    def getStatus(self, el):
        try:
            return el.text.replace("Status:", "")
        except:
            return "N/A"

    def getNumOfChapters(self, el):
        try:
            return len(el)
        except:
            return "N/A"
<|endoftext|>


<commit_before># LIBTBX_SET_DISPATCHER_NAME dev.dials.generate_tutorial_text

from __future__ import division
import os
import shutil
import libtbx.load_env # required for libtbx.env.find_in_repositories
from libtbx.test_utils import open_tmp_directory
from dials.util.procrunner import run_process

class Job(object):
  def __call__(self):
    print self.cmd
    self.result = run_process(self.cmd.split(' '))
    print "running command took {0:.2f} seconds\n".format(self.result['runtime'])
    assert self.result['exitcode'] == 0
    self.mangle_result()
    return {'cmd':self.cmd, 'result':self.result['stdout']}

  def mangle_result(self):
    ''' function that can be overridden to change the return values after execution '''
    pass

class dials_import(Job):

  def __init__(self):
    # find i04 bag training data, this may be part of dials_regression or xia2_regression
    if not libtbx.env.has_module("dials_regression") and not libtbx.env.has_module('xia2_regression'):
      raise RuntimeError("No dials_regression or xia2_regression module available!")

    data_dir = None

    # use the i04_weak_data for this test
    try:
      dials_regression = os.path.join(
        libtbx.env.dist_path('dials_regression'),
        "data", "i04-BAG-training")
      if os.path.isdir(dials_regression): data_dir = dials_regression
    except Exception:
      pass

    xia2_regression = os.path.join(
      abs(libtbx.env.build_path),
      'xia2_regression', 'test_data', 'i04_bag_training')
    if os.path.isdir(xia2_regression): data_dir = xia2_regression

    if data_dir is None:
      raise RuntimeError("Cannot find i04 data in either %s or %s" % (dials_regression, xia2_regression))
    self.cmd = "dials.import {0}".format(os.path.join(data_dir,"th_8_2_0*cbf"))

class dials_find_spots(Job):
  cmd = "dials.find_spots datablock.json nproc=4"

class dials_index(Job):
  cmd = "dials.index datablock.json strong.pickle"

class dials_refine_bravais_settings(Job):
  cmd = "dials.refine_bravais_settings experiments.json indexed.pickle"

class dials_reindex(Job):
  cmd = "dials.reindex indexed.pickle change_of_basis_op=a,b,c"

class dials_refine(Job):
  cmd = "dials.refine bravais_setting_9.json reindexed_reflections.pickle"

class dials_sv_refine(Job):
  cmd = "dials.refine refined_experiments.json refined.pickle scan_varying=true"

class dials_integrate(Job):
  cmd = "dials.integrate refined_experiments.json refined.pickle nproc=4"

class dials_report(Job):
  cmd = "dials.report integrated_experiments.json integrated.pickle"

  def mangle_result(self):
    self.result['stdout'] = open('dials-report.html').read()

class JobWriter(object):
  '''Class to save job command and result to files'''

  def __init__(self, directory):
    self.directory = directory

  def __call__(self, cmd_filename, result_filename, job):
    with open(os.path.join(self.directory, cmd_filename), "w") as f:
      f.write(job['cmd'])
    with open(os.path.join(self.directory, result_filename), "w") as f:
      f.write(job['result'])

if (__name__ == "__main__") :

  cwd = os.path.abspath(os.curdir)
  tmp_dir = open_tmp_directory(suffix="generate_tutorial_text")
  os.chdir(tmp_dir)

  try:
    import_job = dials_import()()

    find_spots_job = dials_find_spots()()

    index_job = dials_index()()

    refine_bravais_settings_job = dials_refine_bravais_settings()()

    reindex_job = dials_reindex()()

    refine_job = dials_refine()()

    sv_refine_job = dials_sv_refine()()

    integrate_job = dials_integrate()()

    report_html_job = dials_report()()

    # if we got this far, assume it is okay to overwrite the logs
    dials_dir = libtbx.env.find_in_repositories("dials")
    result_dir = os.path.join(dials_dir, "doc", "sphinx", "documentation",
                           "tutorials", "logs")

    job_writer = JobWriter(result_dir)
    job_writer("dials.import.cmd", "dials.import.log", import_job)
    job_writer("dials.find_spots.cmd", "dials.find_spots.log", find_spots_job)
    job_writer("dials.index.cmd", "dials.index.log", index_job)
    job_writer("dials.refine_bravais_settings.cmd",
               "dials.refine_bravais_settings.log", refine_bravais_settings_job)
    job_writer("dials.refine.cmd","dials.refine.log", sv_refine_job)
    job_writer("dials.integrate.cmd", "dials.integrate.log", integrate_job)
    job_writer("dials-report.cmd", "dials-report.html", report_html_job)

    print "Updated result files written to {0}".format(result_dir)

  finally:
    os.chdir(cwd)
    # clean up tmp dir
    shutil.rmtree(tmp_dir)
<commit_msg>keep track of more jobs: reindex, static refine and export<commit_after># LIBTBX_SET_DISPATCHER_NAME dev.dials.generate_tutorial_text

from __future__ import division
import os
import shutil
import libtbx.load_env # required for libtbx.env.find_in_repositories
from libtbx.test_utils import open_tmp_directory
from dials.util.procrunner import run_process

class Job(object):
  def __call__(self):
    print self.cmd
    self.result = run_process(self.cmd.split(' '))
    print "running command took {0:.2f} seconds\n".format(self.result['runtime'])
    assert self.result['exitcode'] == 0
    self.mangle_result()
    return {'cmd':self.cmd, 'result':self.result['stdout']}

  def mangle_result(self):
    ''' function that can be overridden to change the return values after execution '''
    pass

class dials_import(Job):

  def __init__(self):
    # find i04 bag training data, this may be part of dials_regression or xia2_regression
    if not libtbx.env.has_module("dials_regression") and not libtbx.env.has_module('xia2_regression'):
      raise RuntimeError("No dials_regression or xia2_regression module available!")

    data_dir = None

    # use the i04_weak_data for this test
    try:
      dials_regression = os.path.join(
        libtbx.env.dist_path('dials_regression'),
        "data", "i04-BAG-training")
      if os.path.isdir(dials_regression): data_dir = dials_regression
    except Exception:
      pass

    xia2_regression = os.path.join(
      abs(libtbx.env.build_path),
      'xia2_regression', 'test_data', 'i04_bag_training')
    if os.path.isdir(xia2_regression): data_dir = xia2_regression

    if data_dir is None:
      raise RuntimeError("Cannot find i04 data in either %s or %s" % (dials_regression, xia2_regression))
    self.cmd = "dials.import {0}".format(os.path.join(data_dir,"th_8_2_0*cbf"))

class dials_find_spots(Job):
  cmd = "dials.find_spots datablock.json nproc=4"

class dials_index(Job):
  cmd = "dials.index datablock.json strong.pickle"

class dials_refine_bravais_settings(Job):
  cmd = "dials.refine_bravais_settings experiments.json indexed.pickle"

class dials_reindex(Job):
  cmd = "dials.reindex indexed.pickle change_of_basis_op=a,b,c"

class dials_refine(Job):
  cmd = "dials.refine bravais_setting_9.json reindexed_reflections.pickle"

class dials_sv_refine(Job):
  cmd = "dials.refine refined_experiments.json refined.pickle scan_varying=true"

class dials_integrate(Job):
  cmd = "dials.integrate refined_experiments.json refined.pickle nproc=4"

class dials_report(Job):
  cmd = "dials.report integrated_experiments.json integrated.pickle"

  def mangle_result(self):
    self.result['stdout'] = open('dials-report.html').read()

class dials_export(Job):
  cmd = "dials.export integrated.pickle refined_experiments.json mtz.hklout=integrated.mtz"

class JobWriter(object):
  '''Class to save job command and result to files'''

  def __init__(self, directory):
    self.directory = directory

  def __call__(self, cmd_filename, result_filename, job):
    with open(os.path.join(self.directory, cmd_filename), "w") as f:
      f.write(job['cmd'])
    with open(os.path.join(self.directory, result_filename), "w") as f:
      f.write(job['result'])

if (__name__ == "__main__") :

  cwd = os.path.abspath(os.curdir)
  tmp_dir = open_tmp_directory(suffix="generate_tutorial_text")
  os.chdir(tmp_dir)

  try:
    import_job = dials_import()()

    find_spots_job = dials_find_spots()()

    index_job = dials_index()()

    refine_bravais_settings_job = dials_refine_bravais_settings()()

    reindex_job = dials_reindex()()

    refine_job = dials_refine()()

    sv_refine_job = dials_sv_refine()()

    integrate_job = dials_integrate()()

    report_html_job = dials_report()()

    export_job = dials_export()()

    # if we got this far, assume it is okay to overwrite the logs
    dials_dir = libtbx.env.find_in_repositories("dials")
    result_dir = os.path.join(dials_dir, "doc", "sphinx", "documentation",
                           "tutorials", "logs")

    job_writer = JobWriter(result_dir)
    job_writer("dials.import.cmd", "dials.import.log", import_job)
    job_writer("dials.find_spots.cmd", "dials.find_spots.log", find_spots_job)
    job_writer("dials.index.cmd", "dials.index.log", index_job)
    job_writer("dials.refine_bravais_settings.cmd",
               "dials.refine_bravais_settings.log", refine_bravais_settings_job)
    job_writer("dials.reindex.cmd", "dials.reindex.log", reindex_job)
    job_writer("dials.refine.cmd","dials.refine.log", refine_job)
    job_writer("dials.sv_refine.cmd","dials.sv_refine.log", sv_refine_job)
    job_writer("dials.integrate.cmd", "dials.integrate.log", integrate_job)
    job_writer("dials-report.cmd", "dials-report.html", report_html_job)
    job_writer("dials-export.cmd", "dials.export.log", export_job)

    print "Updated result files written to {0}".format(result_dir)

  finally:
    os.chdir(cwd)
    # clean up tmp dir
    shutil.rmtree(tmp_dir)
<|endoftext|>


<commit_before>from lxml import etree
import pkg_resources
import logging

from xmodule.modulestore import Location
from functools import partial

log = logging.getLogger('mitx.' + __name__)

def dummy_track(event_type, event):
    pass

class ModuleMissingError(Exception):
    pass

class Plugin(object):
    """
    Base class for a system that uses entry_points to load plugins.

    Implementing classes are expected to have the following attributes:

        entry_point: The name of the entry point to load plugins from
    """

    _plugin_cache = None

    @classmethod
    def load_class(cls, identifier, default=None):
        """
        Loads a single class instance specified by identifier. If identifier
        specifies more than a single class, then logs a warning and returns the first
        class identified.

        If default is not None, will return default if no entry_point matching identifier
        is found. Otherwise, will raise a ModuleMissingError
        """
        if cls._plugin_cache is None:
            cls._plugin_cache = {}

        if identifier not in cls._plugin_cache:
            identifier = identifier.lower()
            classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier))
            if len(classes) > 1:
                log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format(
                    entry_point=cls.entry_point,
                    id=identifier,
                    classes=", ".join(class_.module_name for class_ in classes)))

            if len(classes) == 0:
                if default is not None:
                    return default
                raise ModuleMissingError(identifier)

            cls._plugin_cache[identifier] = classes[0].load()
        return cls._plugin_cache[identifier]

    @classmethod
    def load_classes(cls):
        return [class_.load()
                for class_
                in pkg_resources.iter_entry_points(cls.entry_point)]

class XModule(object):
    ''' Implements a generic learning module.

        Subclasses must at a minimum provide a definition for get_html in order to be displayed to users.

        See the HTML module for a simple example.
    '''

    # The default implementation of get_icon_class returns the icon_class attribute of the class
    # This attribute can be overridden by subclasses, and the function can also be overridden
    # if the icon class depends on the data in the module
    icon_class = 'other'

    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
        '''
        Construct a new xmodule

        system: An I4xSystem allowing access to external resources
        location: Something Location-like that identifies this xmodule
        definition: A dictionary containing 'data' and 'children'. Both are optional
            'data': is JSON-like (string, dictionary, list, bool, or None, optionally nested).
                This defines all of the data necessary for a problem to display that is intrinsic to the problem.
                It should not include any data that would vary between two courses using the same problem
                (due dates, grading policy, randomization, etc.)
            'children': is a list of Location-like values for child modules that this module depends on
        instance_state: A string of serialized json that contains the state of this module for
            current student accessing the system, or None if no state has been saved
        shared_state: A string of serialized json that contains the state that is shared between
            this module and any modules of the same type with the same shared_state_key. This
            state is only shared per-student, not across different students
        kwargs: Optional arguments. Subclasses should always accept kwargs and pass them
            to the parent class constructor.
            Current known uses of kwargs:
                metadata: A dictionary containing data that specifies information that is particular
                    to a problem in the context of a course
        '''
        self.system = system
        self.location = Location(location)
        self.definition = definition
        self.instance_state = instance_state
        self.shared_state = shared_state
        self.id = self.location.url()
        self.name = self.location.name
        self.category = self.location.category
        self.metadata = kwargs.get('metadata', {})
        self._loaded_children = None

    def get_name(self):
        name = self.__xmltree.get('name')
        if name:
            return name
        else:
            raise "We should iterate through children and find a default name"

    def get_children(self):
        '''
        Return module instances for all the children of this module.
        '''
        if self._loaded_children is None:
            self._loaded_children = [self.system.get_module(child) for child in self.definition.get('children', [])]
        return self._loaded_children

    def get_display_items(self):
        '''
        Returns a list of descendent module instances that will display immediately
        inside this module
        '''
        items = []
        for child in self.get_children():
            items.extend(child.displayable_items())

        return items

    def displayable_items(self):
        '''
        Returns list of displayable modules contained by this module. If this module
        is visible, should return [self]
        '''
        return [self]

    def get_icon_class(self):
        '''
        Return a css class identifying this module in the context of an icon
        '''
        return self.icon_class

    ### Functions used in the LMS

    def get_instance_state(self):
        ''' State of the object, as stored in the database
        '''
        return '{}'

    def get_shared_state(self):
        '''
        Get state that should be shared with other instances
        using the same 'shared_state_key' attribute.
        '''
        return '{}'

    def get_score(self):
        ''' Score the student received on the problem.
        '''
        return None

    def max_score(self):
        ''' Maximum score. Two notes:
            * This is generic; in abstract, a problem could be 3/5 points on one randomization, and 5/7 on another
            * In practice, this is a Very Bad Idea, and (a) will break some code in place (although that code
              should get fixed), and (b) break some analytics we plan to put in place.
        '''
        return None

    def get_html(self):
        ''' HTML, as shown in the browser. This is the only method that must be implemented
        '''
        raise NotImplementedError("get_html must be defined for all XModules that appear on the screen. Not defined in %s" % self.__class__.__name__)

    def get_progress(self):
        ''' Return a progress.Progress object that represents how far the student has gone
        in this module.  Must be implemented to get correct progress tracking behavior in
        nesting modules like sequence and vertical.

        If this module has no notion of progress, return None.
        '''
        return None

    def handle_ajax(self, dispatch, get):
        ''' dispatch is last part of the URL.
            get is a dictionary-like object '''
        return ""

class XModuleDescriptor(Plugin):
    """
    An XModuleDescriptor is a specification for an element of a course. This could
    be a problem, an organizational element (a group of content), or a segment of video,
    for example.

    XModuleDescriptors are independent and agnostic to the current student state on a
    problem. They handle the editing interface used by instructors to create a problem,
    and can generate XModules (which do know about student state).
    """
    entry_point = "xmodule.v1"
    js = {}
    js_module = None

    # A list of metadata that this module can inherit from its parent module
    inheritable_metadata = (
        'graded', 'due', 'graceperiod', 'showanswer', 'rerandomize',

        # This is used by the XMLModuleStore to provide for locations for static files,
        # and will need to be removed when that code is removed
        'data_dir'
    )

    # A list of descriptor attributes that must be equal for the discriptors to be
    # equal
    equality_attributes = ('definition', 'metadata', 'location', 'shared_state_key', '_inherited_metadata')

    # ============================= STRUCTURAL MANIPULATION ===========================
    def __init__(self,
                 system,
                 definition=None,
                 **kwargs):
        """
        Construct a new XModuleDescriptor. The only required arguments are the
        system, used for interaction with external resources, and the definition,
        which specifies all the data needed to edit and display the problem (but none
        of the associated metadata that handles recordkeeping around the problem).

        This allows for maximal flexibility to add to the interface while preserving
        backwards compatibility.

        system: An XModuleSystem for interacting with external resources
        definition: A dict containing `data` and `children` representing the problem definition

        Current arguments passed in kwargs:
            location: A xmodule.modulestore.Location object indicating the name and ownership of this problem
            shared_state_key: The key to use for sharing StudentModules with other
                modules of this type
            metadata: A dictionary containing the following optional keys:
                goals: A list of strings of learning goals associated with this module
                display_name: The name to use for displaying this module to the user
                format: The format of this module ('Homework', 'Lab', etc)
                graded (bool): Whether this module is should be graded or not
                due (string): The due date for this module
                graceperiod (string): The amount of grace period to allow when enforcing the due date
                showanswer (string): When to show answers for this module
                rerandomize (string): When to generate a newly randomized instance of the module data
        """
        self.system = system
        self.metadata = kwargs.get('metadata', {})
        self.definition = definition if definition is not None else {}
        self.location = Location(kwargs.get('location'))
        self.name = self.location.name
        self.category = self.location.category
        self.shared_state_key = kwargs.get('shared_state_key')

        self._child_instances = None
        self._inherited_metadata = set()

    def inherit_metadata(self, metadata):
        """
        Updates this module with metadata inherited from a containing module.
        Only metadata specified in self.inheritable_metadata will
        be inherited
        """
        # Set all inheritable metadata from kwargs that are
        # in self.inheritable_metadata and aren't already set in metadata
        for attr in self.inheritable_metadata:
            if attr not in self.metadata and attr in metadata:
                self._inherited_metadata.add(attr)
                self.metadata[attr] = metadata[attr]

    def get_children(self):
        """Returns a list of XModuleDescriptor instances for the children of this module"""
        if self._child_instances is None:
            self._child_instances = []
            for child_loc in self.definition.get('children', []):
                child = self.system.load_item(child_loc)
                child.inherit_metadata(self.metadata)
                self._child_instances.append(child)

        return self._child_instances

    def xmodule_constructor(self, system):
        """
        Returns a constructor for an XModule. This constructor takes two arguments:
        instance_state and shared_state, and returns a fully nstantiated XModule
        """
        return partial(
            self.module_class,
            system,
            self.location,
            self.definition,
            metadata=self.metadata
        )

    # ================================= JSON PARSING ===================================
    @staticmethod
    def load_from_json(json_data, system, default_class=None):
        """
        This method instantiates the correct subclass of XModuleDescriptor based
        on the contents of json_data.

        json_data must contain a 'location' element, and must be suitable to be
        passed into the subclasses `from_json` method.
        """
        class_ = XModuleDescriptor.load_class(
            json_data['location']['category'],
            default_class
        )
        return class_.from_json(json_data, system)

    @classmethod
    def from_json(cls, json_data, system):
        """
        Creates an instance of this descriptor from the supplied json_data.
        This may be overridden by subclasses

        json_data: A json object specifying the definition and any optional keyword arguments for
            the XModuleDescriptor
        system: An XModuleSystem for interacting with external resources
        """
        return cls(system=system, **json_data)

    # ================================= XML PARSING ====================================
    @staticmethod
    def load_from_xml(xml_data,
            system,
            org=None,
            course=None,
            default_class=None):
        """
        This method instantiates the correct subclass of XModuleDescriptor based
        on the contents of xml_data.

        xml_data must be a string containing valid xml
        system is an XMLParsingSystem
        org and course are optional strings that will be used in the generated modules
            url identifiers
        """
        class_ = XModuleDescriptor.load_class(
            etree.fromstring(xml_data).tag,
            default_class
        )
        # leave next line in code, commented out - useful for low-level debugging
        # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (etree.fromstring(xml_data).tag,class_))
        return class_.from_xml(xml_data, system, org, course)

    @classmethod
    def from_xml(cls, xml_data, system, org=None, course=None):
        """
        Creates an instance of this descriptor from the supplied xml_data.
        This may be overridden by subclasses

        xml_data: A string of xml that will be translated into data and children for
            this module
        system is an XMLParsingSystem
        org and course are optional strings that will be used in the generated modules
            url identifiers
        """
        raise NotImplementedError('Modules must implement from_xml to be parsable from xml')

    def export_to_xml(self, resource_fs):
        """
        Returns an xml string representing this module, and all modules underneath it.
        May also write required resources out to resource_fs

        Assumes that modules have single parantage (that no module appears twice in the same course),
        and that it is thus safe to nest modules as xml children as appropriate.

        The returned XML should be able to be parsed back into an identical XModuleDescriptor
        using the from_xml method with the same system, org, and course
        """
        raise NotImplementedError('Modules must implement export_to_xml to enable xml export')

    # ================================== HTML INTERFACE DEFINITIONS ======================
    @classmethod
    def get_javascript(cls):
        """
        Return a dictionary containing some of the following keys:
            coffee: A list of coffeescript fragments that should be compiled and
                    placed on the page
            js: A list of javascript fragments that should be included on the page

        All of these will be loaded onto the page in the CMS
        """
        return cls.js

    def js_module_name(self):
        """
        Return the name of the javascript class to instantiate when
        this module descriptor is loaded for editing
        """
        return self.js_module

    def get_html(self):
        """
        Return the html used to edit this module
        """
        raise NotImplementedError("get_html() must be provided by specific modules")

    # =============================== BUILTIN METHODS ===========================
    def __eq__(self, other):
        eq = (self.__class__ == other.__class__ and
                all(getattr(self, attr, None) == getattr(other, attr, None)
                    for attr in self.equality_attributes))

        if not eq:
            for attr in self.equality_attributes:
                print getattr(self, attr, None), getattr(other, attr, None), getattr(self, attr, None) == getattr(other, attr, None)

        return eq

    def __repr__(self):
        return "{class_}({system!r}, {definition!r}, location={location!r}, metadata={metadata!r})".format(
            class_=self.__class__.__name__,
            system=self.system,
            definition=self.definition,
            location=self.location,
            metadata=self.metadata
        )

class DescriptorSystem(object):
    def __init__(self, load_item, resources_fs, **kwargs):
        """
        load_item: Takes a Location and returns an XModuleDescriptor
        resources_fs: A Filesystem object that contains all of the
            resources needed for the course
        """

        self.load_item = load_item
        self.resources_fs = resources_fs

class XMLParsingSystem(DescriptorSystem):
    def __init__(self, load_item, resources_fs, process_xml, **kwargs):
        """
        process_xml: Takes an xml string, and returns the the XModuleDescriptor created from that xml
        """
        DescriptorSystem.__init__(self, load_item, resources_fs)
        self.process_xml = process_xml
<commit_msg>Add note that metadata dictionary is scaffolding<commit_after>from lxml import etree
import pkg_resources
import logging

from xmodule.modulestore import Location
from functools import partial

log = logging.getLogger('mitx.' + __name__)

def dummy_track(event_type, event):
    pass

class ModuleMissingError(Exception):
    pass

class Plugin(object):
    """
    Base class for a system that uses entry_points to load plugins.

    Implementing classes are expected to have the following attributes:

        entry_point: The name of the entry point to load plugins from
    """

    _plugin_cache = None

    @classmethod
    def load_class(cls, identifier, default=None):
        """
        Loads a single class instance specified by identifier. If identifier
        specifies more than a single class, then logs a warning and returns the first
        class identified.

        If default is not None, will return default if no entry_point matching identifier
        is found. Otherwise, will raise a ModuleMissingError
        """
        if cls._plugin_cache is None:
            cls._plugin_cache = {}

        if identifier not in cls._plugin_cache:
            identifier = identifier.lower()
            classes = list(pkg_resources.iter_entry_points(cls.entry_point, name=identifier))
            if len(classes) > 1:
                log.warning("Found multiple classes for {entry_point} with identifier {id}: {classes}. Returning the first one.".format(
                    entry_point=cls.entry_point,
                    id=identifier,
                    classes=", ".join(class_.module_name for class_ in classes)))

            if len(classes) == 0:
                if default is not None:
                    return default
                raise ModuleMissingError(identifier)

            cls._plugin_cache[identifier] = classes[0].load()
        return cls._plugin_cache[identifier]

    @classmethod
    def load_classes(cls):
        return [class_.load()
                for class_
                in pkg_resources.iter_entry_points(cls.entry_point)]

class XModule(object):
    ''' Implements a generic learning module.

        Subclasses must at a minimum provide a definition for get_html in order to be displayed to users.

        See the HTML module for a simple example.
    '''

    # The default implementation of get_icon_class returns the icon_class attribute of the class
    # This attribute can be overridden by subclasses, and the function can also be overridden
    # if the icon class depends on the data in the module
    icon_class = 'other'

    def __init__(self, system, location, definition, instance_state=None, shared_state=None, **kwargs):
        '''
        Construct a new xmodule

        system: An I4xSystem allowing access to external resources
        location: Something Location-like that identifies this xmodule
        definition: A dictionary containing 'data' and 'children'. Both are optional
            'data': is JSON-like (string, dictionary, list, bool, or None, optionally nested).
                This defines all of the data necessary for a problem to display that is intrinsic to the problem.
                It should not include any data that would vary between two courses using the same problem
                (due dates, grading policy, randomization, etc.)
            'children': is a list of Location-like values for child modules that this module depends on
        instance_state: A string of serialized json that contains the state of this module for
            current student accessing the system, or None if no state has been saved
        shared_state: A string of serialized json that contains the state that is shared between
            this module and any modules of the same type with the same shared_state_key. This
            state is only shared per-student, not across different students
        kwargs: Optional arguments. Subclasses should always accept kwargs and pass them
            to the parent class constructor.
            Current known uses of kwargs:
                metadata: SCAFFOLDING - This dictionary will be split into several different types of metadata
                    in the future (course policy, modification history, etc).
                    A dictionary containing data that specifies information that is particular
                    to a problem in the context of a course
        '''
        self.system = system
        self.location = Location(location)
        self.definition = definition
        self.instance_state = instance_state
        self.shared_state = shared_state
        self.id = self.location.url()
        self.name = self.location.name
        self.category = self.location.category
        self.metadata = kwargs.get('metadata', {})
        self._loaded_children = None

    def get_name(self):
        name = self.__xmltree.get('name')
        if name:
            return name
        else:
            raise "We should iterate through children and find a default name"

    def get_children(self):
        '''
        Return module instances for all the children of this module.
        '''
        if self._loaded_children is None:
            self._loaded_children = [self.system.get_module(child) for child in self.definition.get('children', [])]
        return self._loaded_children

    def get_display_items(self):
        '''
        Returns a list of descendent module instances that will display immediately
        inside this module
        '''
        items = []
        for child in self.get_children():
            items.extend(child.displayable_items())

        return items

    def displayable_items(self):
        '''
        Returns list of displayable modules contained by this module. If this module
        is visible, should return [self]
        '''
        return [self]

    def get_icon_class(self):
        '''
        Return a css class identifying this module in the context of an icon
        '''
        return self.icon_class

    ### Functions used in the LMS

    def get_instance_state(self):
        ''' State of the object, as stored in the database
        '''
        return '{}'

    def get_shared_state(self):
        '''
        Get state that should be shared with other instances
        using the same 'shared_state_key' attribute.
        '''
        return '{}'

    def get_score(self):
        ''' Score the student received on the problem.
        '''
        return None

    def max_score(self):
        ''' Maximum score. Two notes:
            * This is generic; in abstract, a problem could be 3/5 points on one randomization, and 5/7 on another
            * In practice, this is a Very Bad Idea, and (a) will break some code in place (although that code
              should get fixed), and (b) break some analytics we plan to put in place.
        '''
        return None

    def get_html(self):
        ''' HTML, as shown in the browser. This is the only method that must be implemented
        '''
        raise NotImplementedError("get_html must be defined for all XModules that appear on the screen. Not defined in %s" % self.__class__.__name__)

    def get_progress(self):
        ''' Return a progress.Progress object that represents how far the student has gone
        in this module.  Must be implemented to get correct progress tracking behavior in
        nesting modules like sequence and vertical.

        If this module has no notion of progress, return None.
        '''
        return None

    def handle_ajax(self, dispatch, get):
        ''' dispatch is last part of the URL.
            get is a dictionary-like object '''
        return ""

class XModuleDescriptor(Plugin):
    """
    An XModuleDescriptor is a specification for an element of a course. This could
    be a problem, an organizational element (a group of content), or a segment of video,
    for example.

    XModuleDescriptors are independent and agnostic to the current student state on a
    problem. They handle the editing interface used by instructors to create a problem,
    and can generate XModules (which do know about student state).
    """
    entry_point = "xmodule.v1"
    js = {}
    js_module = None

    # A list of metadata that this module can inherit from its parent module
    inheritable_metadata = (
        'graded', 'due', 'graceperiod', 'showanswer', 'rerandomize',

        # This is used by the XMLModuleStore to provide for locations for static files,
        # and will need to be removed when that code is removed
        'data_dir'
    )

    # A list of descriptor attributes that must be equal for the discriptors to be
    # equal
    equality_attributes = ('definition', 'metadata', 'location', 'shared_state_key', '_inherited_metadata')

    # ============================= STRUCTURAL MANIPULATION ===========================
    def __init__(self,
                 system,
                 definition=None,
                 **kwargs):
        """
        Construct a new XModuleDescriptor. The only required arguments are the
        system, used for interaction with external resources, and the definition,
        which specifies all the data needed to edit and display the problem (but none
        of the associated metadata that handles recordkeeping around the problem).

        This allows for maximal flexibility to add to the interface while preserving
        backwards compatibility.

        system: An XModuleSystem for interacting with external resources
        definition: A dict containing `data` and `children` representing the problem definition

        Current arguments passed in kwargs:
            location: A xmodule.modulestore.Location object indicating the name and ownership of this problem
            shared_state_key: The key to use for sharing StudentModules with other
                modules of this type
            metadata: A dictionary containing the following optional keys:
                goals: A list of strings of learning goals associated with this module
                display_name: The name to use for displaying this module to the user
                format: The format of this module ('Homework', 'Lab', etc)
                graded (bool): Whether this module is should be graded or not
                due (string): The due date for this module
                graceperiod (string): The amount of grace period to allow when enforcing the due date
                showanswer (string): When to show answers for this module
                rerandomize (string): When to generate a newly randomized instance of the module data
        """
        self.system = system
        self.metadata = kwargs.get('metadata', {})
        self.definition = definition if definition is not None else {}
        self.location = Location(kwargs.get('location'))
        self.name = self.location.name
        self.category = self.location.category
        self.shared_state_key = kwargs.get('shared_state_key')

        self._child_instances = None
        self._inherited_metadata = set()

    def inherit_metadata(self, metadata):
        """
        Updates this module with metadata inherited from a containing module.
        Only metadata specified in self.inheritable_metadata will
        be inherited
        """
        # Set all inheritable metadata from kwargs that are
        # in self.inheritable_metadata and aren't already set in metadata
        for attr in self.inheritable_metadata:
            if attr not in self.metadata and attr in metadata:
                self._inherited_metadata.add(attr)
                self.metadata[attr] = metadata[attr]

    def get_children(self):
        """Returns a list of XModuleDescriptor instances for the children of this module"""
        if self._child_instances is None:
            self._child_instances = []
            for child_loc in self.definition.get('children', []):
                child = self.system.load_item(child_loc)
                child.inherit_metadata(self.metadata)
                self._child_instances.append(child)

        return self._child_instances

    def xmodule_constructor(self, system):
        """
        Returns a constructor for an XModule. This constructor takes two arguments:
        instance_state and shared_state, and returns a fully nstantiated XModule
        """
        return partial(
            self.module_class,
            system,
            self.location,
            self.definition,
            metadata=self.metadata
        )

    # ================================= JSON PARSING ===================================
    @staticmethod
    def load_from_json(json_data, system, default_class=None):
        """
        This method instantiates the correct subclass of XModuleDescriptor based
        on the contents of json_data.

        json_data must contain a 'location' element, and must be suitable to be
        passed into the subclasses `from_json` method.
        """
        class_ = XModuleDescriptor.load_class(
            json_data['location']['category'],
            default_class
        )
        return class_.from_json(json_data, system)

    @classmethod
    def from_json(cls, json_data, system):
        """
        Creates an instance of this descriptor from the supplied json_data.
        This may be overridden by subclasses

        json_data: A json object specifying the definition and any optional keyword arguments for
            the XModuleDescriptor
        system: An XModuleSystem for interacting with external resources
        """
        return cls(system=system, **json_data)

    # ================================= XML PARSING ====================================
    @staticmethod
    def load_from_xml(xml_data,
            system,
            org=None,
            course=None,
            default_class=None):
        """
        This method instantiates the correct subclass of XModuleDescriptor based
        on the contents of xml_data.

        xml_data must be a string containing valid xml
        system is an XMLParsingSystem
        org and course are optional strings that will be used in the generated modules
            url identifiers
        """
        class_ = XModuleDescriptor.load_class(
            etree.fromstring(xml_data).tag,
            default_class
        )
        # leave next line in code, commented out - useful for low-level debugging
        # log.debug('[XModuleDescriptor.load_from_xml] tag=%s, class_=%s' % (etree.fromstring(xml_data).tag,class_))
        return class_.from_xml(xml_data, system, org, course)

    @classmethod
    def from_xml(cls, xml_data, system, org=None, course=None):
        """
        Creates an instance of this descriptor from the supplied xml_data.
        This may be overridden by subclasses

        xml_data: A string of xml that will be translated into data and children for
            this module
        system is an XMLParsingSystem
        org and course are optional strings that will be used in the generated modules
            url identifiers
        """
        raise NotImplementedError('Modules must implement from_xml to be parsable from xml')

    def export_to_xml(self, resource_fs):
        """
        Returns an xml string representing this module, and all modules underneath it.
        May also write required resources out to resource_fs

        Assumes that modules have single parantage (that no module appears twice in the same course),
        and that it is thus safe to nest modules as xml children as appropriate.

        The returned XML should be able to be parsed back into an identical XModuleDescriptor
        using the from_xml method with the same system, org, and course
        """
        raise NotImplementedError('Modules must implement export_to_xml to enable xml export')

    # ================================== HTML INTERFACE DEFINITIONS ======================
    @classmethod
    def get_javascript(cls):
        """
        Return a dictionary containing some of the following keys:
            coffee: A list of coffeescript fragments that should be compiled and
                    placed on the page
            js: A list of javascript fragments that should be included on the page

        All of these will be loaded onto the page in the CMS
        """
        return cls.js

    def js_module_name(self):
        """
        Return the name of the javascript class to instantiate when
        this module descriptor is loaded for editing
        """
        return self.js_module

    def get_html(self):
        """
        Return the html used to edit this module
        """
        raise NotImplementedError("get_html() must be provided by specific modules")

    # =============================== BUILTIN METHODS ===========================
    def __eq__(self, other):
        eq = (self.__class__ == other.__class__ and
                all(getattr(self, attr, None) == getattr(other, attr, None)
                    for attr in self.equality_attributes))

        if not eq:
            for attr in self.equality_attributes:
                print getattr(self, attr, None), getattr(other, attr, None), getattr(self, attr, None) == getattr(other, attr, None)

        return eq

    def __repr__(self):
        return "{class_}({system!r}, {definition!r}, location={location!r}, metadata={metadata!r})".format(
            class_=self.__class__.__name__,
            system=self.system,
            definition=self.definition,
            location=self.location,
            metadata=self.metadata
        )

class DescriptorSystem(object):
    def __init__(self, load_item, resources_fs, **kwargs):
        """
        load_item: Takes a Location and returns an XModuleDescriptor
        resources_fs: A Filesystem object that contains all of the
            resources needed for the course
        """

        self.load_item = load_item
        self.resources_fs = resources_fs

class XMLParsingSystem(DescriptorSystem):
    def __init__(self, load_item, resources_fs, process_xml, **kwargs):
        """
        process_xml: Takes an xml string, and returns the the XModuleDescriptor created from that xml
        """
        DescriptorSystem.__init__(self, load_item, resources_fs)
        self.process_xml = process_xml
<|endoftext|>


<commit_before>#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Snapshot Build Bisect Tool

This script bisects a snapshot archive using binary search. It starts at
a bad revision (it will try to guess HEAD) and asks for a last known-good
revision. It will then binary search across this revision range by downloading,
unzipping, and opening Chromium for you. After testing the specific revision,
it will ask you whether it is good or bad before continuing the search.
"""

# Base URL to download snapshots from.
BUILD_BASE_URL = 'http://build.chromium.org/buildbot/snapshots/'

# The type (platform) of the build archive. This is what's passed in to the
# '-a/--archive' option.
BUILD_ARCHIVE_TYPE = ''

# The selected archive to bisect.
BUILD_ARCHIVE_DIR = ''

# The location of the builds.
BUILD_ARCHIVE_URL = '/%d/'

# Name of the build archive.
BUILD_ZIP_NAME = ''

# Directory name inside the archive.
BUILD_DIR_NAME = ''

# Name of the executable.
BUILD_EXE_NAME = ''

# URL to the ViewVC commit page.
BUILD_VIEWVC_URL = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%d'

# Changelogs URL
CHANGELOG_URL = 'http://build.chromium.org/buildbot/' \
                'perf/dashboard/ui/changelog.html?url=/trunk/src&range=%d:%d'

###############################################################################

import math
import optparse
import os
import pipes
import re
import shutil
import sys
import tempfile
import urllib
import zipfile

def UnzipFilenameToDir(filename, dir):
  """Unzip |filename| to directory |dir|."""
  zf = zipfile.ZipFile(filename)
  # Make base.
  pushd = os.getcwd()
  try:
    if not os.path.isdir(dir):
      os.mkdir(dir)
    os.chdir(dir)
    # Extract files.
    for info in zf.infolist():
      name = info.filename
      if name.endswith('/'):  # dir
        if not os.path.isdir(name):
          os.makedirs(name)
      else:  # file
        dir = os.path.dirname(name)
        if not os.path.isdir(dir):
          os.makedirs(dir)
        out = open(name, 'wb')
        out.write(zf.read(name))
        out.close()
      # Set permissions. Permission info in external_attr is shifted 16 bits.
      os.chmod(name, info.external_attr >> 16L)
    os.chdir(pushd)
  except Exception, e:
    print >>sys.stderr, e
    sys.exit(1)

def SetArchiveVars(archive):
  """Set a bunch of global variables appropriate for the specified archive."""
  global BUILD_ARCHIVE_TYPE
  global BUILD_ARCHIVE_DIR
  global BUILD_ZIP_NAME
  global BUILD_DIR_NAME
  global BUILD_EXE_NAME
  global BUILD_BASE_URL

  BUILD_ARCHIVE_TYPE = archive
  BUILD_ARCHIVE_DIR = 'chromium-rel-' + BUILD_ARCHIVE_TYPE

  if BUILD_ARCHIVE_TYPE in ('linux', 'linux-64', 'linux-chromiumos'):
    BUILD_ZIP_NAME = 'chrome-linux.zip'
    BUILD_DIR_NAME = 'chrome-linux'
    BUILD_EXE_NAME = 'chrome'
  elif BUILD_ARCHIVE_TYPE in ('mac'):
    BUILD_ZIP_NAME = 'chrome-mac.zip'
    BUILD_DIR_NAME = 'chrome-mac'
    BUILD_EXE_NAME = 'Chromium.app/Contents/MacOS/Chromium'
  elif BUILD_ARCHIVE_TYPE in ('xp'):
    BUILD_ZIP_NAME = 'chrome-win32.zip'
    BUILD_DIR_NAME = 'chrome-win32'
    BUILD_EXE_NAME = 'chrome.exe'

  BUILD_BASE_URL += BUILD_ARCHIVE_DIR

def ParseDirectoryIndex(url):
  """Parses the HTML directory listing into a list of revision numbers."""
  handle = urllib.urlopen(url)
  dirindex = handle.read()
  handle.close()
  return re.findall(r'\1/', dirindex)

def GetRevList(good, bad):
  """Gets the list of revision numbers between |good| and |bad|."""
  # Download the main revlist.
  revlist = ParseDirectoryIndex(BUILD_BASE_URL)
  revlist = map(int, revlist)
  revlist = filter(lambda r: range(good, bad).__contains__(int(r)), revlist)
  revlist.sort()
  return revlist

def TryRevision(rev, profile, args):
  """Downloads revision |rev|, unzips it, and opens it for the user to test.
  |profile| is the profile to use."""
  # Do this in a temp dir so we don't collide with user files.
  cwd = os.getcwd()
  tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
  os.chdir(tempdir)

  # Download the file.
  download_url = BUILD_BASE_URL + (BUILD_ARCHIVE_URL % rev) + BUILD_ZIP_NAME
  try:
    print 'Fetching ' + download_url
    urllib.urlretrieve(download_url, BUILD_ZIP_NAME)
  except Exception, e:
    print('Could not retrieve the download. Sorry.')
    sys.exit(-1)

  # Unzip the file.
  print 'Unziping ...'
  UnzipFilenameToDir(BUILD_ZIP_NAME, os.curdir)

  # Tell the system to open the app.
  args = ['--user-data-dir=%s' % profile] + args
  flags = ' '.join(map(pipes.quote, args))
  exe = os.path.join(os.getcwd(), BUILD_DIR_NAME, BUILD_EXE_NAME)
  cmd = '%s %s' % (exe, flags)
  print 'Running %s' % cmd
  os.system(cmd)

  os.chdir(cwd)
  print 'Cleaning temp dir ...'
  try:
    shutil.rmtree(tempdir, True)
  except Exception, e:
    pass

def AskIsGoodBuild(rev):
  """Ask the user whether build |rev| is good or bad."""
  # Loop until we get a response that we can parse.
  while True:
    response = raw_input('\nBuild %d is [(g)ood/(b)ad]: ' % int(rev))
    if response and response in ('g', 'b'):
      return response == 'g'

def main():
  usage = ('%prog [options] [-- chromium-options]\n'
           'Perform binary search on the snapshot builds.\n'
           '\n'
           'Tip: add "-- --no-first-run" to bypass the first run prompts.')
  parser = optparse.OptionParser(usage=usage)
  # Strangely, the default help output doesn't include the choice list.
  choices = ['mac', 'xp', 'linux', 'linux-64', 'linux-chromiumos']
  parser.add_option('-a', '--archive',
                    choices = choices,
                    help = 'The buildbot archive to bisect [%s].' %
                           '|'.join(choices))
  parser.add_option('-b', '--bad', type = 'int',
                    help = 'The bad revision to bisect to.')
  parser.add_option('-g', '--good', type = 'int',
                    help = 'The last known good revision to bisect from.')
  parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
                    help = 'Profile to use; this will not reset every run. ' +
                    'Defaults to a clean profile.')
  (opts, args) = parser.parse_args()

  if opts.archive is None:
    print 'Error: missing required parameter: --archive'
    print
    parser.print_help()
    return 1

  if opts.bad and opts.good and (opts.good > opts.bad):
    print ('The good revision (%d) must precede the bad revision (%d).\n' %
           (opts.good, opts.bad))
    parser.print_help()
    return 1

  SetArchiveVars(opts.archive)

  # Pick a starting point, try to get HEAD for this.
  if opts.bad:
    bad_rev = opts.bad
  else:
    bad_rev = 0
    try:
      # Location of the latest build revision number
      BUILD_LATEST_URL = '%s/LATEST' % (BUILD_BASE_URL)
      nh = urllib.urlopen(BUILD_LATEST_URL)
      latest = int(nh.read())
      nh.close()
      bad_rev = raw_input('Bad revision [HEAD:%d]: ' % latest)
      if (bad_rev == ''):
        bad_rev = latest
      bad_rev = int(bad_rev)
    except Exception, e:
      print('Could not determine latest revision. This could be bad...')
      bad_rev = int(raw_input('Bad revision: '))

  # Find out when we were good.
  if opts.good:
    good_rev = opts.good
  else:
    good_rev = 0
    try:
      good_rev = int(raw_input('Last known good [0]: '))
    except Exception, e:
      pass

  # Get a list of revisions to bisect across.
  revlist = GetRevList(good_rev, bad_rev)
  if len(revlist) < 2:  # Don't have enough builds to bisect
    print 'We don\'t have enough builds to bisect. revlist: %s' % revlist
    sys.exit(1)

  # If we don't have a |good_rev|, set it to be the first revision possible.
  if good_rev == 0:
    good_rev = revlist[0]

  # These are indexes of |revlist|.
  good = 0
  bad = len(revlist) - 1
  last_known_good_rev = revlist[good]

  # Binary search time!
  while good < bad:
    candidates = revlist[good:bad]
    num_poss = len(candidates)
    if num_poss > 10:
      print('%d candidates. %d tries left.' %
          (num_poss, round(math.log(num_poss, 2))))
    else:
      print('Candidates: %s' % revlist[good:bad])

    # Cut the problem in half...
    test = int((bad - good) / 2) + good
    test_rev = revlist[test]

    # Let the user give this rev a spin (in her own profile, if she wants).
    profile = opts.profile
    if not profile:
      profile = 'profile'  # In a temp dir.
    TryRevision(test_rev, profile, args)
    if AskIsGoodBuild(test_rev):
      last_known_good_rev = revlist[good]
      good = test + 1
    else:
      bad = test

  # We're done. Let the user know the results in an official manner.
  print('You are probably looking for build %d.' % revlist[bad])
  print('CHANGELOG URL:')
  print(CHANGELOG_URL % (last_known_good_rev, revlist[bad]))
  print('Built at revision:')
  print(BUILD_VIEWVC_URL % revlist[bad])

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Let build-bisect.py print download progress.<commit_after>#!/usr/bin/python
# Copyright (c) 2010 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Snapshot Build Bisect Tool

This script bisects a snapshot archive using binary search. It starts at
a bad revision (it will try to guess HEAD) and asks for a last known-good
revision. It will then binary search across this revision range by downloading,
unzipping, and opening Chromium for you. After testing the specific revision,
it will ask you whether it is good or bad before continuing the search.
"""

# Base URL to download snapshots from.
BUILD_BASE_URL = 'http://build.chromium.org/buildbot/snapshots/'

# The type (platform) of the build archive. This is what's passed in to the
# '-a/--archive' option.
BUILD_ARCHIVE_TYPE = ''

# The selected archive to bisect.
BUILD_ARCHIVE_DIR = ''

# The location of the builds.
BUILD_ARCHIVE_URL = '/%d/'

# Name of the build archive.
BUILD_ZIP_NAME = ''

# Directory name inside the archive.
BUILD_DIR_NAME = ''

# Name of the executable.
BUILD_EXE_NAME = ''

# URL to the ViewVC commit page.
BUILD_VIEWVC_URL = 'http://src.chromium.org/viewvc/chrome?view=rev&revision=%d'

# Changelogs URL
CHANGELOG_URL = 'http://build.chromium.org/buildbot/' \
                'perf/dashboard/ui/changelog.html?url=/trunk/src&range=%d:%d'

###############################################################################

import math
import optparse
import os
import pipes
import re
import shutil
import sys
import tempfile
import urllib
import zipfile

def UnzipFilenameToDir(filename, dir):
  """Unzip |filename| to directory |dir|."""
  zf = zipfile.ZipFile(filename)
  # Make base.
  pushd = os.getcwd()
  try:
    if not os.path.isdir(dir):
      os.mkdir(dir)
    os.chdir(dir)
    # Extract files.
    for info in zf.infolist():
      name = info.filename
      if name.endswith('/'):  # dir
        if not os.path.isdir(name):
          os.makedirs(name)
      else:  # file
        dir = os.path.dirname(name)
        if not os.path.isdir(dir):
          os.makedirs(dir)
        out = open(name, 'wb')
        out.write(zf.read(name))
        out.close()
      # Set permissions. Permission info in external_attr is shifted 16 bits.
      os.chmod(name, info.external_attr >> 16L)
    os.chdir(pushd)
  except Exception, e:
    print >>sys.stderr, e
    sys.exit(1)

def SetArchiveVars(archive):
  """Set a bunch of global variables appropriate for the specified archive."""
  global BUILD_ARCHIVE_TYPE
  global BUILD_ARCHIVE_DIR
  global BUILD_ZIP_NAME
  global BUILD_DIR_NAME
  global BUILD_EXE_NAME
  global BUILD_BASE_URL

  BUILD_ARCHIVE_TYPE = archive
  BUILD_ARCHIVE_DIR = 'chromium-rel-' + BUILD_ARCHIVE_TYPE

  if BUILD_ARCHIVE_TYPE in ('linux', 'linux-64', 'linux-chromiumos'):
    BUILD_ZIP_NAME = 'chrome-linux.zip'
    BUILD_DIR_NAME = 'chrome-linux'
    BUILD_EXE_NAME = 'chrome'
  elif BUILD_ARCHIVE_TYPE in ('mac'):
    BUILD_ZIP_NAME = 'chrome-mac.zip'
    BUILD_DIR_NAME = 'chrome-mac'
    BUILD_EXE_NAME = 'Chromium.app/Contents/MacOS/Chromium'
  elif BUILD_ARCHIVE_TYPE in ('xp'):
    BUILD_ZIP_NAME = 'chrome-win32.zip'
    BUILD_DIR_NAME = 'chrome-win32'
    BUILD_EXE_NAME = 'chrome.exe'

  BUILD_BASE_URL += BUILD_ARCHIVE_DIR

def ParseDirectoryIndex(url):
  """Parses the HTML directory listing into a list of revision numbers."""
  handle = urllib.urlopen(url)
  dirindex = handle.read()
  handle.close()
  return re.findall(r'\1/', dirindex)

def GetRevList(good, bad):
  """Gets the list of revision numbers between |good| and |bad|."""
  # Download the main revlist.
  revlist = ParseDirectoryIndex(BUILD_BASE_URL)
  revlist = map(int, revlist)
  revlist = filter(lambda r: range(good, bad).__contains__(int(r)), revlist)
  revlist.sort()
  return revlist

def TryRevision(rev, profile, args):
  """Downloads revision |rev|, unzips it, and opens it for the user to test.
  |profile| is the profile to use."""
  # Do this in a temp dir so we don't collide with user files.
  cwd = os.getcwd()
  tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
  os.chdir(tempdir)

  # Download the file.
  download_url = BUILD_BASE_URL + (BUILD_ARCHIVE_URL % rev) + BUILD_ZIP_NAME
  def _Reporthook(blocknum, blocksize, totalsize):
    size = blocknum * blocksize
    if totalsize == -1:  # Total size not known.
      progress = "Received %d bytes" % size
    else:
      size = min(totalsize, size)
      progress = "Received %d of %d bytes, %.2f%%" % (
          size, totalsize, 100.0 * size / totalsize)
    # Send a \r to let all progress messages use just one line of output.
    sys.stdout.write("\r" + progress)
    sys.stdout.flush()
  try:
    print 'Fetching ' + download_url
    urllib.urlretrieve(download_url, BUILD_ZIP_NAME, _Reporthook)
    print
  except Exception, e:
    print('Could not retrieve the download. Sorry.')
    sys.exit(-1)

  # Unzip the file.
  print 'Unziping ...'
  UnzipFilenameToDir(BUILD_ZIP_NAME, os.curdir)

  # Tell the system to open the app.
  args = ['--user-data-dir=%s' % profile] + args
  flags = ' '.join(map(pipes.quote, args))
  exe = os.path.join(os.getcwd(), BUILD_DIR_NAME, BUILD_EXE_NAME)
  cmd = '%s %s' % (exe, flags)
  print 'Running %s' % cmd
  os.system(cmd)

  os.chdir(cwd)
  print 'Cleaning temp dir ...'
  try:
    shutil.rmtree(tempdir, True)
  except Exception, e:
    pass

def AskIsGoodBuild(rev):
  """Ask the user whether build |rev| is good or bad."""
  # Loop until we get a response that we can parse.
  while True:
    response = raw_input('\nBuild %d is [(g)ood/(b)ad]: ' % int(rev))
    if response and response in ('g', 'b'):
      return response == 'g'

def main():
  usage = ('%prog [options] [-- chromium-options]\n'
           'Perform binary search on the snapshot builds.\n'
           '\n'
           'Tip: add "-- --no-first-run" to bypass the first run prompts.')
  parser = optparse.OptionParser(usage=usage)
  # Strangely, the default help output doesn't include the choice list.
  choices = ['mac', 'xp', 'linux', 'linux-64', 'linux-chromiumos']
  parser.add_option('-a', '--archive',
                    choices = choices,
                    help = 'The buildbot archive to bisect [%s].' %
                           '|'.join(choices))
  parser.add_option('-b', '--bad', type = 'int',
                    help = 'The bad revision to bisect to.')
  parser.add_option('-g', '--good', type = 'int',
                    help = 'The last known good revision to bisect from.')
  parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
                    help = 'Profile to use; this will not reset every run. ' +
                    'Defaults to a clean profile.')
  (opts, args) = parser.parse_args()

  if opts.archive is None:
    print 'Error: missing required parameter: --archive'
    print
    parser.print_help()
    return 1

  if opts.bad and opts.good and (opts.good > opts.bad):
    print ('The good revision (%d) must precede the bad revision (%d).\n' %
           (opts.good, opts.bad))
    parser.print_help()
    return 1

  SetArchiveVars(opts.archive)

  # Pick a starting point, try to get HEAD for this.
  if opts.bad:
    bad_rev = opts.bad
  else:
    bad_rev = 0
    try:
      # Location of the latest build revision number
      BUILD_LATEST_URL = '%s/LATEST' % (BUILD_BASE_URL)
      nh = urllib.urlopen(BUILD_LATEST_URL)
      latest = int(nh.read())
      nh.close()
      bad_rev = raw_input('Bad revision [HEAD:%d]: ' % latest)
      if (bad_rev == ''):
        bad_rev = latest
      bad_rev = int(bad_rev)
    except Exception, e:
      print('Could not determine latest revision. This could be bad...')
      bad_rev = int(raw_input('Bad revision: '))

  # Find out when we were good.
  if opts.good:
    good_rev = opts.good
  else:
    good_rev = 0
    try:
      good_rev = int(raw_input('Last known good [0]: '))
    except Exception, e:
      pass

  # Get a list of revisions to bisect across.
  revlist = GetRevList(good_rev, bad_rev)
  if len(revlist) < 2:  # Don't have enough builds to bisect
    print 'We don\'t have enough builds to bisect. revlist: %s' % revlist
    sys.exit(1)

  # If we don't have a |good_rev|, set it to be the first revision possible.
  if good_rev == 0:
    good_rev = revlist[0]

  # These are indexes of |revlist|.
  good = 0
  bad = len(revlist) - 1
  last_known_good_rev = revlist[good]

  # Binary search time!
  while good < bad:
    candidates = revlist[good:bad]
    num_poss = len(candidates)
    if num_poss > 10:
      print('%d candidates. %d tries left.' %
          (num_poss, round(math.log(num_poss, 2))))
    else:
      print('Candidates: %s' % revlist[good:bad])

    # Cut the problem in half...
    test = int((bad - good) / 2) + good
    test_rev = revlist[test]

    # Let the user give this rev a spin (in her own profile, if she wants).
    profile = opts.profile
    if not profile:
      profile = 'profile'  # In a temp dir.
    TryRevision(test_rev, profile, args)
    if AskIsGoodBuild(test_rev):
      last_known_good_rev = revlist[good]
      good = test + 1
    else:
      bad = test

  # We're done. Let the user know the results in an official manner.
  print('You are probably looking for build %d.' % revlist[bad])
  print('CHANGELOG URL:')
  print(CHANGELOG_URL % (last_known_good_rev, revlist[bad]))
  print('Built at revision:')
  print(BUILD_VIEWVC_URL % revlist[bad])

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

"""
Contains classes used during the build process to collect files
into BuildPackages.  These BuildPackages are used to create Packages
and create changesets from the files created during the build process
"""

import stat
import string
import re
import os
import files

def BuildDeviceFile(devtype, major, minor, owner, group, perms):
    if devtype == "b":
	f = files.BlockDevice(None)
    elif devtype == "c":
	f = files.CharacterDevice(None)
    else:
	raise AssertionError

    f.devt.setMajor(major)
    f.devt.setMinor(minor)
    f.inode.setOwner(owner)
    f.inode.setGroup(group)
    f.inode.setPerms(perms)
    f.inode.setMtime(0)
    f.flags.set(0)

    return f

class BuildPackage(dict):

    def addFile(self, path, realPath):
        """
        Add a file to the build package

        @param path: the destination of the file in the package
        @param realPath: the location of the actual file on the filesystem,
        used to obtain the contents of the file when creating a changeset
        to commit to the repository
        """
	f = files.FileFromFilesystem(realPath, None)
	f.inode.setPerms(f.inode.perms() & 01777)
	# blech.
	f.realPath = realPath
	self[path] = f

    def addDevice(self, path, devtype, major, minor,
                  owner='root', group='root', perms=0660):
        """
        Add a device node to the build package

        @param path: the destination of the device node in the package
        """
        f = BuildDeviceFile(devtype, major, minor, owner, group, perms)
	# blech.
	f.realPath = None
	self[path] = f

    def getName(self):
        """
        Return the name of the BuildPackage

        @returns: name of the BuildPackag
        @rtype: str
        """
	return self.name

    def getVersion(self):
        """
        Return the version of the BuildPackage

        @returns: name of the BuildPackag
        @rtype: versions.Version instance
        """
	return self.version

    def __init__(self, name, version):
	self.name = name
	self.version = version
	dict.__init__(self)

class AutoBuildPackage:
    """
    AutoBuildPackage creates a set of BuildPackage instances and
    provides facilities for automatically populating them with files
    according to Filters.
    """
    def __init__(self, version, pkgFilters, compFilters):
        """
	@param version: the version of each package
        @type version: versions.Version instance
	@param pkgFilters: Filters used to add files to main packages
	@type pkgFilters: sequence of Filter instances
	@param compFilters: Filters used to add files to components
	@type compFilters: sequence of Filter instances
	"""
	self.pkgFilters = pkgFilters
        self.compFilters = compFilters
        # dictionary of all the build packages
        self.packages = {}
        # reverse map from the package:component combination to
        # the correct build package
	self.packageMap = {}
	for main in self.pkgFilters:
	    for comp in self.compFilters:
		name = self._getname(main.name, comp.name)
		if name not in self.packages:
		    self.packages[name] = BuildPackage(name, version)
		if main not in self.packageMap:
		    self.packageMap[main] = {}
		self.packageMap[main][comp] = self.packages[name]
	# dictionary from pathnames to fileobjects
	self.pathMap = {}
	# dictionary from pathnames to packages
	self.pkgMap = {}

    def _getname(self, pkgname, compname):
        return string.join((pkgname, compname), ':')

    def findPackage(self, path):
        """Return the BuildPackage that matches the path"""
	for main in self.pkgFilters:
	    if main.match(path):
		for comp in self.compFilters:
		    if comp.match(path):
			return self.packageMap[main][comp]
        return None

    def addFile(self, path, realPath):
        """
        Add a path to the correct BuildPackage instance by matching
        the file name against the package and component filters

        @param path: path to add to the BuildPackage
        @type path: str
        @rtype: None
        """
        pkg = self.findPackage(path)
        pkg.addFile(path, realPath)
	self.pathMap[path] = pkg[path]
	self.pkgMap[path] = pkg

    def addDevice(self, path, devtype, major, minor,
                  owner='root', group='root', perms=0660):
        """
        Add a device to the correct BuildPackage instance by matching
        the file name against the package and component filters
        """
        pkg = self.findPackage(path)
        pkg.addDevice(path, devtype, major, minor, owner, group, perms)
	self.pathMap[path] = pkg[path]
	self.pkgMap[path] = pkg

    def getPackages(self):
        """
        Examine the BuildPackage instances that have been created and
        return a list that includes only those which have files

        @return: list of BuildPackages instances
        @rtype: list
        """
        l = []
        for name in self.packages.keys():
            if self.packages[name].keys():
                l.append(self.packages[name])
        return l

    def walk(self, root):
        """
        Traverse the directory tree specified by @C{root}, adding file
        entries to the BuildPackages

        @param root: root of path to walk
        @type root: str
        @rtype: None
        """
        os.path.walk(root, _autoVisit, (root, self))

def _autoVisit(arg, dir, files):
    """
    Helper function called by os.path.walk() when AutoBuildPackage.walk()
    is called
    """
    (root, autopkg) = arg
    dir = dir[len(root):]

    for file in files:
        if dir:
            path = dir + '/' + file
        else:
            path = '/' + file

        autopkg.addFile(path, root + path)
<commit_msg>remove unused imports<commit_after>#
# Copyright (c) 2004 Specifix, Inc.
# All rights reserved
#

"""
Contains classes used during the build process to collect files
into BuildPackages.  These BuildPackages are used to create Packages
and create changesets from the files created during the build process
"""

import string
import os
import files

def BuildDeviceFile(devtype, major, minor, owner, group, perms):
    if devtype == "b":
	f = files.BlockDevice(None)
    elif devtype == "c":
	f = files.CharacterDevice(None)
    else:
	raise AssertionError

    f.devt.setMajor(major)
    f.devt.setMinor(minor)
    f.inode.setOwner(owner)
    f.inode.setGroup(group)
    f.inode.setPerms(perms)
    f.inode.setMtime(0)
    f.flags.set(0)

    return f

class BuildPackage(dict):

    def addFile(self, path, realPath):
        """
        Add a file to the build package

        @param path: the destination of the file in the package
        @param realPath: the location of the actual file on the filesystem,
        used to obtain the contents of the file when creating a changeset
        to commit to the repository
        """
	f = files.FileFromFilesystem(realPath, None)
	f.inode.setPerms(f.inode.perms() & 01777)
	# blech.
	f.realPath = realPath
	self[path] = f

    def addDevice(self, path, devtype, major, minor,
                  owner='root', group='root', perms=0660):
        """
        Add a device node to the build package

        @param path: the destination of the device node in the package
        """
        f = BuildDeviceFile(devtype, major, minor, owner, group, perms)
	# blech.
	f.realPath = None
	self[path] = f

    def getName(self):
        """
        Return the name of the BuildPackage

        @returns: name of the BuildPackag
        @rtype: str
        """
	return self.name

    def getVersion(self):
        """
        Return the version of the BuildPackage

        @returns: name of the BuildPackag
        @rtype: versions.Version instance
        """
	return self.version

    def __init__(self, name, version):
	self.name = name
	self.version = version
	dict.__init__(self)

class AutoBuildPackage:
    """
    AutoBuildPackage creates a set of BuildPackage instances and
    provides facilities for automatically populating them with files
    according to Filters.
    """
    def __init__(self, version, pkgFilters, compFilters):
        """
	@param version: the version of each package
        @type version: versions.Version instance
	@param pkgFilters: Filters used to add files to main packages
	@type pkgFilters: sequence of Filter instances
	@param compFilters: Filters used to add files to components
	@type compFilters: sequence of Filter instances
	"""
	self.pkgFilters = pkgFilters
        self.compFilters = compFilters
        # dictionary of all the build packages
        self.packages = {}
        # reverse map from the package:component combination to
        # the correct build package
	self.packageMap = {}
	for main in self.pkgFilters:
	    for comp in self.compFilters:
		name = self._getname(main.name, comp.name)
		if name not in self.packages:
		    self.packages[name] = BuildPackage(name, version)
		if main not in self.packageMap:
		    self.packageMap[main] = {}
		self.packageMap[main][comp] = self.packages[name]
	# dictionary from pathnames to fileobjects
	self.pathMap = {}
	# dictionary from pathnames to packages
	self.pkgMap = {}

    def _getname(self, pkgname, compname):
        return string.join((pkgname, compname), ':')

    def findPackage(self, path):
        """Return the BuildPackage that matches the path"""
	for main in self.pkgFilters:
	    if main.match(path):
		for comp in self.compFilters:
		    if comp.match(path):
			return self.packageMap[main][comp]
        return None

    def addFile(self, path, realPath):
        """
        Add a path to the correct BuildPackage instance by matching
        the file name against the package and component filters

        @param path: path to add to the BuildPackage
        @type path: str
        @rtype: None
        """
        pkg = self.findPackage(path)
        pkg.addFile(path, realPath)
	self.pathMap[path] = pkg[path]
	self.pkgMap[path] = pkg

    def addDevice(self, path, devtype, major, minor,
                  owner='root', group='root', perms=0660):
        """
        Add a device to the correct BuildPackage instance by matching
        the file name against the package and component filters
        """
        pkg = self.findPackage(path)
        pkg.addDevice(path, devtype, major, minor, owner, group, perms)
	self.pathMap[path] = pkg[path]
	self.pkgMap[path] = pkg

    def getPackages(self):
        """
        Examine the BuildPackage instances that have been created and
        return a list that includes only those which have files

        @return: list of BuildPackages instances
        @rtype: list
        """
        l = []
        for name in self.packages.keys():
            if self.packages[name].keys():
                l.append(self.packages[name])
        return l

    def walk(self, root):
        """
        Traverse the directory tree specified by @C{root}, adding file
        entries to the BuildPackages

        @param root: root of path to walk
        @type root: str
        @rtype: None
        """
        os.path.walk(root, _autoVisit, (root, self))

def _autoVisit(arg, dir, files):
    """
    Helper function called by os.path.walk() when AutoBuildPackage.walk()
    is called
    """
    (root, autopkg) = arg
    dir = dir[len(root):]

    for file in files:
        if dir:
            path = dir + '/' + file
        else:
            path = '/' + file

        autopkg.addFile(path, root + path)
<|endoftext|>