<commit_before>import wsgiref.handlers
import logging
import re
import os

from google.appengine.api import quota
from google.appengine.api import urlfetch
from google.appengine.api import memcache
from google.appengine.api import datastore_errors
from google.appengine.api.urlfetch import DownloadError
from google.appengine.api.labs import taskqueue
from google.appengine.api.taskqueue import Task
from google.appengine.ext import webapp
from google.appengine.ext import db
from google.appengine.ext.webapp import template
from google.appengine.ext.db import GeoPt

from google.appengine.runtime import apiproxy_errors

from BeautifulSoup import BeautifulSoup, Tag

from data_model import RouteListing
from data_model import StopLocation
from data_model import DeveloperKeys

URLBASE = "http://webwatch.cityofmadison.com/webwatch/ada.aspx?"
CRAWL_URLBASE = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"

class CrawlerHandler(webapp.RequestHandler):
    def get(self):
        # create a new task with this link
        crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
        task = Task(url='/crawlingtask', params={'crawl':crawlURL,'routeID':'00'})
        task.add('crawler')
        logging.info("Added new task for %s" % crawlURL)
        return

## end CrawlerHandler()

class CrawlingTaskHandler(webapp.RequestHandler):
    def post(self):
        try:
            scrapeURL = self.request.get('crawl')
            direction = self.request.get('direction')
            routeID = self.request.get('routeID')
            logging.debug("task scraping for %s, direction %s, route %s" % (scrapeURL,direction,routeID))

            loop = 0
            done = False
            result = None
            start = quota.get_request_cpu_usage()
            while not done and loop < 3:
                try:
                    # fetch the page
                    result = urlfetch.fetch(scrapeURL)
                    done = True;
                except urlfetch.DownloadError:
                    logging.info("Error loading page (%s)... sleeping" % loop)
                    if result:
                        logging.debug("Error status: %s" % result.status_code)
                        logging.debug("Error header: %s" % result.headers)
                        logging.debug("Error content: %s" % result.content)
                        time.sleep(4)
                        loop = loop+1
            end = quota.get_request_cpu_usage()
            #logging.info("scraping took %s cycles" % (end-start))

            # start to interrogate the results
            soup = BeautifulSoup(result.content)
            stopUpdates = []
            for slot in soup.html.body.findAll("a","ada"):
                #logging.info("pulling out data from page... %s" % slot)

                if slot.has_key('href'):
                    href = slot['href']
                    title = slot['title']
                    logging.info("FOUND A TITLE ----> %s" % title)
                    # route crawler looks for titles with an ID# string
                    if title.find("[ID#") > 0:
                        # we finally got down to the page we're looking for

                        # pull the stopID from the page content...
                        stopID = title.split("ID#")[1].split("]")[0]

                        # pull the intersection from the page content...
                        intersection = title.split("[")[0].strip()

                        logging.info("found stop %s, %s" % (stopID,intersection))

                        # check for conflicts...
                        q = db.GqlQuery("SELECT * FROM StopLocation WHERE stopID = :1 and direction = :2", stopID,direction.upper())
                        stopQuery = q.fetch(1)
                        if len(stopQuery) == 0:
                            # add the new stop
                            stop = StopLocation()
                            stop.stopID = stopID
                            stop.routeID = routeID
                            stop.intersection = intersection.upper()
                            stop.direction = direction.upper()
                            stopUpdates.append(stop)  # stop.put()
                            logging.info("added new stop listing MINUS geo location")
                        else:
                            logging.info("already have this stop in the table...")
                            stopQuery[0].routeID = routeID
                            stopUpdates.append(stopQuery[0])

                        # pull the route and direction data from the URL
                        #routeData = scrapeURL.split('?')[1]
                        #logging.info("found the page! arguments: %s stopID: %s" % (routeData,stopID))
                        #routeArgs = routeData.split('&')
                        #routeID = routeArgs[0].split('=')[1]
                        #directionID = routeArgs[1].split('=')[1]
                        #timeEstimatesURL = CRAWL_URLBASE + href

                        # check for conflicts...
                        #q = db.GqlQuery("SELECT * FROM RouteListing WHERE route = :1 AND direction = :2 AND stopID = :3",
                                        #routeID, directionID, stopID)
                        #routeQuery = q.fetch(1)
                        #if len(routeQuery) == 0:
                          # add the new route to the DB
                          #route = RouteListing()
                          #route.route = routeID
                          #route.direction = directionID
                          #route.stopID = stopID
                          #route.scheduleURL = timeEstimatesURL
                          #route.put()
                          #logging.info("added new route listing entry to the database!")
                        #else:
                          #logging.error("we found a duplicate entry!?! %s", routeQuery[0].scheduleURL)
                    #else: # title.split(",")[0].isdigit():
                    elif href.find("?r=") > -1:
                        # create a new task with this link
                        crawlURL = CRAWL_URLBASE + href
                        if routeID == '00' or len(routeID) > 2:
                            routeID = href.split('r=')[1]
                        task = Task(url='/crawlingtask', params={'crawl':crawlURL,'direction':title,'routeID':routeID})
                        task.add('crawler')
                        logging.info("Added new task for %s, direction %s, route %s" % (title.split(",")[0],title,routeID))
                    # label crawler looks for titles with letters for extraction/persistence
                    #elif title.replace('-','').replace(' ','').isalpha():
                    #    routeData = href.split('?')[1]
                    #    logging.info("found the route LABEL page! href: %s" % href)
                    #    routeArgs = routeData.split('&')
                    #    directionID = routeArgs[1].split('=')[1]
                    #
                    #    l = DestinationListing.get_or_insert(title, id=directionID, label=title)

            # push the vehicle updates to the datastore
            db.put(stopUpdates)

        except apiproxy_errors.DeadlineExceededError:
            logging.error("DeadlineExceededError exception!?")
            return

        return;

## end CrawlingTask()

class DropTableHandler(webapp.RequestHandler):
    def post(self, model=""):
        logging.debug('HEADS UP -> we have a request to drop the entities in the %s model' % model)
        self.get(model)

    def get(self, model=""):
        qstring = "select * from " + model
        logging.info("query string is... %s" % qstring)
        q = db.GqlQuery(qstring)
        results = q.fetch(500)
        while results:
            if model == 'RouteListing':
                for r in results:
                    try:
                        sLocation = r.stopLocation
                    except datastore_errors.Error,e:
                        if e.args[0] == "ReferenceProperty failed to be resolved":
                            sLocation = None
                        else:
                            raise
                    if sLocation is not None:
                        r.stopLocation = None
                        r.put()
            else:
                db.delete(results)

            results = q.fetch(500, len(results))

## end

class StartTableDrop(webapp.RequestHandler):
    def get(self,model=""):
        url = '/droptable/%s' % model
        logging.error('getting ready to start task to drop the %s table' % model)
        task = Task(url=url, params={})
        task.add('crawler')
        self.response.out.write('got it. started the background task to delete all %s entities' % model)

class CreateDeveloperKeysHandler(webapp.RequestHandler):
    def get(self):
      key = DeveloperKeys()
      key.developerName = 'tester'
      key.developerKey = 'mfoolskiosk'
      key.developerEmail = 'gtracy@gmail.com'
      key.requestCounter = 0
      key.errorCounter = 0
      key.put()

## end

class RouteListHandler(webapp.RequestHandler):
    def get(self,routeID=""):
      logging.info("fetching all stop locations for route %s" % routeID)
      q = db.GqlQuery("SELECT * FROM StopLocation WHERE routeID = :1", routeID)
      if q is not None:
          results = []

          # Perform the query to get 500 results.
          stops = q.fetch(500)
          logging.info("running through stop location list....")
          for s in stops:
              logging.debug("fetch route list details for stop %s" % s.stopID)
#              try:
#                  sLocation = s.stopLocation
#              except datastore_errors.Error,e:
#                  if e.args[0] == "ReferenceProperty failed to be resolved":
#                      sLocation = None
#                  else:
#                      raise

              results.append({'stopID':s.stopID,
                              #'location':sLocation if sLocation is not None else 'unknown',
                              'intersection':s.intersection,# if sLocation is not None else 'unknown',
                              'direction':s.direction,# if sLocation is not None else 'unknown',
                              'routeID':routeID,
                              })

      # add the counter to the template values
      template_values = {'stops':results}

      # create a page that provides a form for sending an SMS message
      path = os.path.join(os.path.dirname(__file__), 'stop.html')
      self.response.out.write(template.render(path,template_values))

## end

## end ScrapeRouteStopHandler
def main():
  logging.getLogger().setLevel(logging.DEBUG)
  application = webapp.WSGIApplication([('/configure', CrawlerHandler),
                                        ('/crawlingtask', CrawlingTaskHandler),
                                        ('/routelist/(.*)', RouteListHandler),
                                        ('/droptable/(.*)', DropTableHandler),
                                        ('/debug/drop/(.*)', StartTableDrop),
                                        ('/debug/create/newkey', CreateDeveloperKeysHandler),
                                        ],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)

if __name__ == '__main__':
  main()

<commit_msg>hacking on the crawler to include route 82 in the route listing<commit_after>import wsgiref.handlers
import logging
import re
import os

from google.appengine.api import quota
from google.appengine.api import urlfetch
from google.appengine.api import memcache
from google.appengine.api import datastore_errors
from google.appengine.api.urlfetch import DownloadError
from google.appengine.api.labs import taskqueue
from google.appengine.api.taskqueue import Task
from google.appengine.ext import webapp
from google.appengine.ext import db
from google.appengine.ext.webapp import template
from google.appengine.ext.db import GeoPt

from google.appengine.runtime import apiproxy_errors

from BeautifulSoup import BeautifulSoup, Tag

from data_model import RouteListing
from data_model import StopLocation
from data_model import DeveloperKeys

URLBASE = "http://webwatch.cityofmadison.com/webwatch/ada.aspx?"
CRAWL_URLBASE = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"

class CrawlerHandler(webapp.RequestHandler):
    def get(self,routeID=""):
        # create a new task with this link
        #crawlURL = "http://webwatch.cityofmadison.com/webwatch/Ada.aspx"
        crawlURL = URLBASE + 'r=' + routeID
        task = Task(url='/routelist/crawlingtask', params={'crawl':crawlURL,'routeID':'00'})
        task.add('crawler')
        logging.info("Added new task for %s" % crawlURL)
        return

## end CrawlerHandler()

class CrawlingTaskHandler(webapp.RequestHandler):
    def post(self):
        try:
            scrapeURL = self.request.get('crawl')
            direction = self.request.get('direction')
            routeID = self.request.get('routeID')
            logging.debug("task scraping for %s, direction %s, route %s" % (scrapeURL,direction,routeID))

            loop = 0
            done = False
            result = None
            start = quota.get_request_cpu_usage()
            while not done and loop < 3:
                try:
                    # fetch the page
                    result = urlfetch.fetch(scrapeURL)
                    done = True;
                except urlfetch.DownloadError:
                    logging.info("Error loading page (%s)... sleeping" % loop)
                    if result:
                        logging.debug("Error status: %s" % result.status_code)
                        logging.debug("Error header: %s" % result.headers)
                        logging.debug("Error content: %s" % result.content)
                        time.sleep(4)
                        loop = loop+1
            end = quota.get_request_cpu_usage()
            #logging.info("scraping took %s cycles" % (end-start))

            # start to interrogate the results
            soup = BeautifulSoup(result.content)
            stopUpdates = []
            for slot in soup.html.body.findAll("a","ada"):
                logging.info("pulling out data from page... %s" % slot)

                if slot.has_key('href'):
                    href = slot['href']
                    title = slot['title']
                    logging.info("FOUND A TITLE ----> %s" % title)
                    # route crawler looks for titles with an ID# string
                    if title.find("#") > 0:
                        # we finally got down to the page we're looking for

                        # pull the stopID from the page content...
                        stopID = title.split("#")[1].split("]")[0]

                        # pull the intersection from the page content...
                        intersection = title.split("[")[0].strip()

                        logging.info("found stop %s, %s" % (stopID,intersection))

                        # check for conflicts...
                        q = db.GqlQuery("SELECT * FROM StopLocation WHERE stopID = :1 and direction = :2", stopID,direction.upper())
                        stopQuery = q.fetch(1)
                        if len(stopQuery) == 0:
                            # add the new stop
                            stop = StopLocation()
                            stop.stopID = stopID
                            stop.routeID = routeID
                            stop.intersection = intersection.upper()
                            stop.direction = direction.upper()
                            stopUpdates.append(stop)  # stop.put()
                            logging.info("ADDED StopLocation (%s) - MINUS geo location" % stopID)
                        else:
                            logging.info("StopLoation entity already exists for %s..." % stopID)
                            stopQuery[0].routeID = routeID
                            stopUpdates.append(stopQuery[0])

                        # pull the route and direction data from the URL
                        routeData = scrapeURL.split('?')[1]
                        logging.info("FOUND THE PAGE ---> arguments: %s stopID: %s" % (routeData,stopID))
                        routeArgs = routeData.split('&')
                        routeID = routeArgs[0].split('=')[1]
                        directionID = routeArgs[1].split('=')[1]
                        timeEstimatesURL = CRAWL_URLBASE + href

                        # check for conflicts...
                        r = db.GqlQuery("SELECT * FROM RouteListing WHERE route = :1 AND direction = :2 AND stopID = :3",
                                        routeID, directionID, stopID).get()
                        if r is None:
                          # add the new route to the DB
                          route = RouteListing()
                          route.route = routeID
                          route.direction = directionID
                          route.stopID = stopID
                          route.scheduleURL = timeEstimatesURL
                          route.put()
                          logging.info("added new route listing entry to the database!")
                        else:
                          logging.error("we found a duplicate entry!?! %s", r.scheduleURL)
                    #else: # title.split(",")[0].isdigit():
                    elif href.find("?r=") > -1:
                        # create a new task with this link
                        crawlURL = CRAWL_URLBASE + href
                        if routeID == '00':
                            routeID = href.split('r=')[1]
                        elif href.find("&") > -1:
                            routeID = href.split('&')[0].split('r=')[1]
                        task = Task(url='/routelist/crawlingtask', params={'crawl':crawlURL,'direction':title,'routeID':routeID})
                        task.add('crawler')
                        logging.info("Added new task for %s, direction %s, route %s" % (title.split(",")[0],title,routeID))
                    # label crawler looks for titles with letters for extraction/persistence
                    #elif title.replace('-','').replace(' ','').isalpha():
                    #    routeData = href.split('?')[1]
                    #    logging.info("found the route LABEL page! href: %s" % href)
                    #    routeArgs = routeData.split('&')
                    #    directionID = routeArgs[1].split('=')[1]
                    #
                    #    l = DestinationListing.get_or_insert(title, id=directionID, label=title)

            # push the vehicle updates to the datastore
            db.put(stopUpdates)

        except apiproxy_errors.DeadlineExceededError:
            logging.error("DeadlineExceededError exception!?")
            return

        return;

## end CrawlingTask()

class DropTableHandler(webapp.RequestHandler):
    def post(self, model=""):
        logging.debug('HEADS UP -> we have a request to drop the entities in the %s model' % model)
        self.get(model)

    def get(self, model=""):
        qstring = "select * from " + model
        logging.info("query string is... %s" % qstring)
        q = db.GqlQuery(qstring)
        results = q.fetch(500)
        while results:
            if model == 'RouteListing':
                for r in results:
                    try:
                        sLocation = r.stopLocation
                    except datastore_errors.Error,e:
                        if e.args[0] == "ReferenceProperty failed to be resolved":
                            sLocation = None
                        else:
                            raise
                    if sLocation is not None:
                        r.stopLocation = None
                        r.put()
            else:
                db.delete(results)

            results = q.fetch(500, len(results))

## end

class StartTableDrop(webapp.RequestHandler):
    def get(self,model=""):
        url = '/droptable/%s' % model
        logging.error('getting ready to start task to drop the %s table' % model)
        task = Task(url=url, params={})
        task.add('crawler')
        self.response.out.write('got it. started the background task to delete all %s entities' % model)

class CreateDeveloperKeysHandler(webapp.RequestHandler):
    def get(self):
      key = DeveloperKeys()
      key.developerName = 'tester'
      key.developerKey = 'mfoolskiosk'
      key.developerEmail = 'gtracy@gmail.com'
      key.requestCounter = 0
      key.errorCounter = 0
      key.put()

## end

class RouteListHandler(webapp.RequestHandler):
    def get(self,routeID=""):
      logging.info("fetching all stop locations for route %s" % routeID)
      q = db.GqlQuery("SELECT * FROM StopLocation WHERE routeID = :1", routeID)
      if q is not None:
          results = []

          # Perform the query to get 500 results.
          stops = q.fetch(500)
          logging.info("running through stop location list....")
          for s in stops:
              logging.debug("fetch route list details for stop %s" % s.stopID)
#              try:
#                  sLocation = s.stopLocation
#              except datastore_errors.Error,e:
#                  if e.args[0] == "ReferenceProperty failed to be resolved":
#                      sLocation = None
#                  else:
#                      raise

              results.append({'stopID':s.stopID,
                              #'location':sLocation if sLocation is not None else 'unknown',
                              'intersection':s.intersection,# if sLocation is not None else 'unknown',
                              'direction':s.direction,# if sLocation is not None else 'unknown',
                              'routeID':routeID,
                              })

      # add the counter to the template values
      template_values = {'stops':results}

      # create a page that provides a form for sending an SMS message
      path = os.path.join(os.path.dirname(__file__), 'stop.html')
      self.response.out.write(template.render(path,template_values))

## end

## end ScrapeRouteStopHandler
def main():
  logging.getLogger().setLevel(logging.DEBUG)
  application = webapp.WSGIApplication([('/routelist/configure/(.*)', CrawlerHandler),
                                        ('/routelist/crawlingtask', CrawlingTaskHandler),
                                        ('/routelist/(.*)', RouteListHandler),
                                        ('/droptable/(.*)', DropTableHandler),
                                        ('/debug/drop/(.*)', StartTableDrop),
                                        ('/debug/create/newkey', CreateDeveloperKeysHandler),
                                        ],
                                       debug=True)
  wsgiref.handlers.CGIHandler().run(application)

if __name__ == '__main__':
  main()

<|endoftext|>


<commit_before>import re
import itertools

import wifi.subprocess_compat as subprocess
from pbkdf2 import PBKDF2
from wifi.utils import ensure_file_exists
from wifi.exceptions import ConnectionError

def configuration(cell, passkey=None):
    """
    Returns a dictionary of configuration options for cell

    Asks for a password if necessary
    """
    if not cell.encrypted:
        return {
            'wireless-essid': cell.ssid,
            'wireless-channel': 'auto',
        }
    else:
        if cell.encryption_type.startswith('wpa'):
            if len(passkey) != 64:
                passkey = PBKDF2(passkey, cell.ssid, 4096).hexread(32)

            return {
                'wpa-ssid': cell.ssid,
                'wpa-psk': passkey,
                'wireless-channel': 'auto',
            }
        elif cell.encryption_type == 'wep':
            return {
                'wireless-essid': cell.ssid,
                'wireless-key': passkey,
            }
        else:
            raise NotImplementedError

bound_ip_re = re.compile(r'^bound to (?P<ip_address>\S+)', flags=re.MULTILINE)

class Scheme(object):
    """
    Saved configuration for connecting to a wireless network.  This
    class provides a Python interface to the /etc/network/interfaces
    file.
    """

    interfaces = '/etc/network/interfaces'

    @classmethod
    def for_file(cls, interfaces):
        """
        A class factory for providing a nice way to specify the interfaces file
        that you want to use.  Use this instead of directly overwriting the
        interfaces Class attribute if you care about thread safety.
        """
        return type(cls)(cls.__name__, (cls,), {
            'interfaces': interfaces,
        })

    def __init__(self, interface, name, options=None):
        self.interface = interface
        self.name = name
        self.options = options or {}

    def __str__(self):
        """
        Returns the representation of a scheme that you would need
        in the /etc/network/interfaces file.
        """
        iface = "iface {interface}-{name} inet dhcp".format(**vars(self))
        options = ''.join("\n    {k} {v}".format(k=k, v=v) for k, v in self.options.items())
        return iface + options + '\n'

    def __repr__(self):
        return 'Scheme(interface={interface!r}, name={name!r}, options={options!r}'.format(**vars(self))

    @classmethod
    def all(cls):
        """
        Returns an generator of saved schemes.
        """
        ensure_file_exists(cls.interfaces)
        with open(cls.interfaces, 'r') as f:
            return extract_schemes(f.read(), scheme_class=cls)

    @classmethod
    def where(cls, fn):
        return list(filter(fn, cls.all()))

    @classmethod
    def find(cls, interface, name):
        """
        Returns a :class:`Scheme` or `None` based on interface and
        name.
        """
        try:
            return cls.where(lambda s: s.interface == interface and s.name == name)[0]
        except IndexError:
            return None

    @classmethod
    def for_cell(cls, interface, name, cell, passkey=None):
        """
        Intuits the configuration needed for a specific
        :class:`Cell` and creates a :class:`Scheme` for it.
        """
        return cls(interface, name, configuration(cell, passkey))

    def save(self):
        """
        Writes the configuration to the :attr:`interfaces` file.
        """
        assert not self.find(self.interface, self.name), "This scheme already exists"

        with open(self.interfaces, 'a') as f:
            f.write('\n')
            f.write(str(self))

    def delete(self):
        """
        Deletes the configuration from the :attr:`interfaces` file.
        """
        iface = "iface %s-%s inet dhcp" % (self.interface, self.name)
        content = ''
        with open(self.interfaces, 'r') as f:
            skip = False
            for line in f:
                if not line.strip():
                    skip = False
                elif line.strip() == iface:
                    skip = True
                if not skip:
                    content += line
        with open(self.interfaces, 'w') as f:
            f.write(content)

    @property
    def iface(self):
        return '{0}-{1}'.format(self.interface, self.name)

    def as_args(self):
        args = list(itertools.chain.from_iterable(
            ('-o', '{k}={v}'.format(k=k, v=v)) for k, v in self.options.items()))

        return [self.interface + '=' + self.iface] + args

    def activate(self):
        """
        Connects to the network as configured in this scheme.
        """

        subprocess.check_output(['/sbin/ifdown', self.interface], stderr=subprocess.STDOUT)
        ifup_output = subprocess.check_output(['/sbin/ifup'] + self.as_args(), stderr=subprocess.STDOUT)

        return self.parse_ifup_output(ifup_output)

    def parse_ifup_output(self, output):
        matches = bound_ip_re.search(output)
        if matches:
            return Connection(scheme=self, ip_address=matches.group('ip_address'))
        else:
            raise ConnectionError("Failed to connect to %r" % self)

class Connection(object):
    """
    The connection object returned when connecting to a Scheme.
    """
    def __init__(self, scheme, ip_address):
        self.scheme = scheme
        self.ip_address = ip_address

# TODO: support other interfaces
scheme_re = re.compile(r'iface\s+(?P<interface>wlan\d?)(?:-(?P<name>\w+))?')

def extract_schemes(interfaces, scheme_class=Scheme):
    lines = interfaces.splitlines()
    while lines:
        line = lines.pop(0)

        if line.startswith('#') or not line:
            continue

        match = scheme_re.match(line)
        if match:
            options = {}
            interface, scheme = match.groups()

            if not scheme or not interface:
                continue

            while lines and lines[0].startswith(' '):
                key, value = re.sub(r'\s{2,}', ' ', lines.pop(0).strip()).split(' ', 1)
                options[key] = value

            scheme = scheme_class(interface, scheme, options)

            yield scheme
<commit_msg>Fixed Python3 bytes error with Scheme.activate.<commit_after>import re
import itertools

import wifi.subprocess_compat as subprocess
from pbkdf2 import PBKDF2
from wifi.utils import ensure_file_exists
from wifi.exceptions import ConnectionError

def configuration(cell, passkey=None):
    """
    Returns a dictionary of configuration options for cell

    Asks for a password if necessary
    """
    if not cell.encrypted:
        return {
            'wireless-essid': cell.ssid,
            'wireless-channel': 'auto',
        }
    else:
        if cell.encryption_type.startswith('wpa'):
            if len(passkey) != 64:
                passkey = PBKDF2(passkey, cell.ssid, 4096).hexread(32)

            return {
                'wpa-ssid': cell.ssid,
                'wpa-psk': passkey,
                'wireless-channel': 'auto',
            }
        elif cell.encryption_type == 'wep':
            return {
                'wireless-essid': cell.ssid,
                'wireless-key': passkey,
            }
        else:
            raise NotImplementedError

bound_ip_re = re.compile(r'^bound to (?P<ip_address>\S+)', flags=re.MULTILINE)

class Scheme(object):
    """
    Saved configuration for connecting to a wireless network.  This
    class provides a Python interface to the /etc/network/interfaces
    file.
    """

    interfaces = '/etc/network/interfaces'

    @classmethod
    def for_file(cls, interfaces):
        """
        A class factory for providing a nice way to specify the interfaces file
        that you want to use.  Use this instead of directly overwriting the
        interfaces Class attribute if you care about thread safety.
        """
        return type(cls)(cls.__name__, (cls,), {
            'interfaces': interfaces,
        })

    def __init__(self, interface, name, options=None):
        self.interface = interface
        self.name = name
        self.options = options or {}

    def __str__(self):
        """
        Returns the representation of a scheme that you would need
        in the /etc/network/interfaces file.
        """
        iface = "iface {interface}-{name} inet dhcp".format(**vars(self))
        options = ''.join("\n    {k} {v}".format(k=k, v=v) for k, v in self.options.items())
        return iface + options + '\n'

    def __repr__(self):
        return 'Scheme(interface={interface!r}, name={name!r}, options={options!r}'.format(**vars(self))

    @classmethod
    def all(cls):
        """
        Returns an generator of saved schemes.
        """
        ensure_file_exists(cls.interfaces)
        with open(cls.interfaces, 'r') as f:
            return extract_schemes(f.read(), scheme_class=cls)

    @classmethod
    def where(cls, fn):
        return list(filter(fn, cls.all()))

    @classmethod
    def find(cls, interface, name):
        """
        Returns a :class:`Scheme` or `None` based on interface and
        name.
        """
        try:
            return cls.where(lambda s: s.interface == interface and s.name == name)[0]
        except IndexError:
            return None

    @classmethod
    def for_cell(cls, interface, name, cell, passkey=None):
        """
        Intuits the configuration needed for a specific
        :class:`Cell` and creates a :class:`Scheme` for it.
        """
        return cls(interface, name, configuration(cell, passkey))

    def save(self):
        """
        Writes the configuration to the :attr:`interfaces` file.
        """
        assert not self.find(self.interface, self.name), "This scheme already exists"

        with open(self.interfaces, 'a') as f:
            f.write('\n')
            f.write(str(self))

    def delete(self):
        """
        Deletes the configuration from the :attr:`interfaces` file.
        """
        iface = "iface %s-%s inet dhcp" % (self.interface, self.name)
        content = ''
        with open(self.interfaces, 'r') as f:
            skip = False
            for line in f:
                if not line.strip():
                    skip = False
                elif line.strip() == iface:
                    skip = True
                if not skip:
                    content += line
        with open(self.interfaces, 'w') as f:
            f.write(content)

    @property
    def iface(self):
        return '{0}-{1}'.format(self.interface, self.name)

    def as_args(self):
        args = list(itertools.chain.from_iterable(
            ('-o', '{k}={v}'.format(k=k, v=v)) for k, v in self.options.items()))

        return [self.interface + '=' + self.iface] + args

    def activate(self):
        """
        Connects to the network as configured in this scheme.
        """

        subprocess.check_output(['/sbin/ifdown', self.interface], stderr=subprocess.STDOUT)
        ifup_output = subprocess.check_output(['/sbin/ifup'] + self.as_args(), stderr=subprocess.STDOUT)
        ifup_output = ifup_output.decode('utf-8')

        return self.parse_ifup_output(ifup_output)

    def parse_ifup_output(self, output):
        matches = bound_ip_re.search(output)
        if matches:
            return Connection(scheme=self, ip_address=matches.group('ip_address'))
        else:
            raise ConnectionError("Failed to connect to %r" % self)

class Connection(object):
    """
    The connection object returned when connecting to a Scheme.
    """
    def __init__(self, scheme, ip_address):
        self.scheme = scheme
        self.ip_address = ip_address

# TODO: support other interfaces
scheme_re = re.compile(r'iface\s+(?P<interface>wlan\d?)(?:-(?P<name>\w+))?')

def extract_schemes(interfaces, scheme_class=Scheme):
    lines = interfaces.splitlines()
    while lines:
        line = lines.pop(0)

        if line.startswith('#') or not line:
            continue

        match = scheme_re.match(line)
        if match:
            options = {}
            interface, scheme = match.groups()

            if not scheme or not interface:
                continue

            while lines and lines[0].startswith(' '):
                key, value = re.sub(r'\s{2,}', ' ', lines.pop(0).strip()).split(' ', 1)
                options[key] = value

            scheme = scheme_class(interface, scheme, options)

            yield scheme
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""

After running PASA (https://github.com/PASApipeline/PASApipeline) we want to create a 'unigene'
set, which includes the transcript from each PASA cluster with the longest ORF.  This approach
was published in ??

Input expectations:

In the input_pasta_fasta file the IDs are expected to look like this:

   >asmbl_1
   >asmbl_2
   >asmbl_3

Within the GTF, the last column should look like:

   gene_id "PASA_cluster_1"; transcript_id "align_id:907554|asmbl_1";
   gene_id "PASA_cluster_2"; transcript_id "align_id:907555|asmbl_2";
   gene_id "PASA_cluster_19"; transcript_id "align_id:907573|asmbl_20";
   gene_id "PASA_cluster_19"; transcript_id "align_id:907574|asmbl_21";

Since transdecoder is run on the PASA FASTA output, its IDs should match those PASA provided.

Assumes transdecoder output looks like this ('longest_orfs.cds'):

   >asmbl_6.p1 type:5prime_partial len:126 asmbl_6:2055-1678(-)
   >asmbl_6.p2 type:complete len:104 asmbl_6:374-685(+)

"""

import argparse
import os
import re

from biocode import utils

def main():
    parser = argparse.ArgumentParser( description='Export the transcript from each PASA cluster with the longest ORF')
    parser.add_argument('-if', '--input_pasa_fasta', type=str, required=True, help='Path to PASAs predicted FASTA file' )
    parser.add_argument('-itsc', '--input_pasa_transcript_size_cutoff', type=str, required=False, default=300, help='Nucleotide size cutoff of input transcripts' )
    parser.add_argument('-itc', '--input_pasa_tpm_cutoff', type=str, required=False, default=0.1, help='TPM cutoff of input transcripts' )
    parser.add_argument('-ig', '--input_pasa_gtf', type=str, required=True, help='Path to PASAs predicted GFT file' )
    parser.add_argument('-it', '--input_transdecoder', type=str, required=True, help='Path to the longest_orfs.cds from transdecoder' )
    parser.add_argument('-is', '--input_salmon', type=str, required=True, help='Path to quant.sf file from Salmon' )
    parser.add_argument('-ol', '--output_file_longest_orf', type=str, required=True, help='Path to an output file to be created' )
    parser.add_argument('-oa', '--output_file_most_abundant', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    seqs = utils.fasta_dict_from_file(args.input_pasa_fasta)
    pasa_clusters = load_pasa_clusters(args.input_pasa_gtf)
    orf_lengths = longest_orf_lengths(args.input_transdecoder)
    load_abundances(seqs, args.input_salmon)

    ofh_longest = open(args.output_file_longest_orf, 'wt')
    ofh_abund   = open(args.output_file_most_abundant, 'wt')

    pasa_cluster_count = len(pasa_clusters)
    pasa_transcript_count = len(seqs)
    size_filtered_pasa_transcript_count = 0
    size_filtered_pasa_transcript_count_with_cds = 0
    size_filtered_pasa_cluster_count = 0
    tpm_filtered_pasa_transcript_count = 0
    tpm_filtered_pasa_cluster_count = 0

    for cluster_id in pasa_clusters:
        longest_asmbl_id = None
        longest_asmbl_len = 0

        best_tpm_id = None
        best_tpm = 0

        for asmbl_id in pasa_clusters[cluster_id]:
            if asmbl_id in seqs and 'tpm' in seqs[asmbl_id] and seqs[asmbl_id]['tpm'] >= args.input_pasa_tpm_cutoff:
                tpm_filtered_pasa_transcript_count += 1

            if len(seqs[asmbl_id]['s']) >= args.input_pasa_transcript_size_cutoff:
                size_filtered_pasa_transcript_count += 1

                if asmbl_id in orf_lengths and orf_lengths[asmbl_id] > longest_asmbl_len:
                    size_filtered_pasa_transcript_count_with_cds += 1
                    longest_asmbl_id = asmbl_id
                    longest_asmbl_len = orf_lengths[asmbl_id]

                if asmbl_id in seqs and 'tpm' in seqs[asmbl_id] and seqs[asmbl_id]['tpm'] > best_tpm:
                    best_tpm_id = asmbl_id
                    best_tpm = seqs[asmbl_id]['tpm']

        if longest_asmbl_id:
            size_filtered_pasa_cluster_count += 1
            ofh_longest.write(">{0} {1} longest_orf:{3}\n{2}\n".format(longest_asmbl_id, seqs[longest_asmbl_id]['h'],
                                                                       utils.wrapped_fasta(''.join(seqs[longest_asmbl_id]['s'])),
                                                                       longest_asmbl_len))
        if best_tpm_id:
            tpm_filtered_pasa_cluster_count += 1
            ofh_abund.write(">{0} {1} tpm:{3}\n{2}\n".format(best_tpm_id, seqs[best_tpm_id]['h'],
                                                             utils.wrapped_fasta(''.join(seqs[best_tpm_id]['s'])),
                                                             best_tpm))

    ofh_longest.close()
    ofh_abund.close()

    print("Stats:\n")
    print("PASA cluster count: {0}".format(pasa_cluster_count))
    print("PASA transcript count: {0}".format(pasa_transcript_count))
    print("Size-filtered PASA cluster count: {0}".format(size_filtered_pasa_cluster_count))
    print("Size-filtered PASA transcript count: {0}".format(size_filtered_pasa_transcript_count))
    print("Size-filtered PASA transcript count with CDS: {0}".format(size_filtered_pasa_transcript_count_with_cds))
    print("TPM-filtered PASA cluster count: {0}".format(tpm_filtered_pasa_cluster_count))
    print("TPM-filtered PASA transcript count: {0}".format(tpm_filtered_pasa_transcript_count))

def load_abundances(seqs, salmon_file):
    line_num = 0
    for line in open(salmon_file):
        line_num += 1
        if line_num == 1:
            continue

        line = line.rstrip()
        cols = line.split("\t")

        if len(cols) < 5:
            continue

        seq_id = cols[0]
        tpm = float(cols[3])

        if seq_id not in seqs:
            raise Exception("ERROR: Got seq ID ({0}) in salmon input that wasn't in FASTA input".format(seq_id))

        if tpm > 0.1:
            seqs[seq_id]['tpm'] = tpm

def load_pasa_clusters(pasa_gtf):
    clusters = dict()
    for line in open(pasa_gtf):
        line = line.rstrip()
        cols = line.split("\t")

        if len(cols) < 9:
            continue

        if cols[2] != 'transcript':
            continue

        m = re.match("gene_id \"(.+?)\"; transcript_id \"align_id:\d+\|(.+?)\";", cols[8])

        if m:
            cluster_id = m.group(1)
            asmbl_id = m.group(2)

            if cluster_id not in clusters:
                clusters[cluster_id] = dict()

            # initialize all their longest-ORF lengths to zero
            clusters[cluster_id][asmbl_id] = 0

    return clusters

def longest_orf_lengths(transdecoder_file):
    orf_lengths = dict()

    for line in open (transdecoder_file):
        line = line.rstrip()
        m = re.match(">.+? type:\S+ len:(\d+) (asmbl_\d+)\:.+", line)
        if m:
            orf_length = int(m.group(1))
            asmbl_id = m.group(2)

            if asmbl_id in orf_lengths:
                if orf_length > orf_lengths[asmbl_id]:
                    orf_lengths[asmbl_id] = orf_length
            else:
                orf_lengths[asmbl_id] = orf_length

    return orf_lengths

if __name__ == '__main__':
    main()

<commit_msg>Made steps much more clear and arranged output to match re-ordering<commit_after>#!/usr/bin/env python3

"""

After running PASA (https://github.com/PASApipeline/PASApipeline) we want to create a 'unigene'
set, which includes the transcript from each PASA cluster with the longest ORF.  This approach
was published in ??

Input expectations:

In the input_pasta_fasta file the IDs are expected to look like this:

   >asmbl_1
   >asmbl_2
   >asmbl_3

Within the GTF, the last column should look like:

   gene_id "PASA_cluster_1"; transcript_id "align_id:907554|asmbl_1";
   gene_id "PASA_cluster_2"; transcript_id "align_id:907555|asmbl_2";
   gene_id "PASA_cluster_19"; transcript_id "align_id:907573|asmbl_20";
   gene_id "PASA_cluster_19"; transcript_id "align_id:907574|asmbl_21";

Since transdecoder is run on the PASA FASTA output, its IDs should match those PASA provided.

Assumes transdecoder output looks like this ('longest_orfs.cds'):

   >asmbl_6.p1 type:5prime_partial len:126 asmbl_6:2055-1678(-)
   >asmbl_6.p2 type:complete len:104 asmbl_6:374-685(+)

"""

import argparse
import os
import re
import sys

from biocode import utils

def main():
    parser = argparse.ArgumentParser( description='Export the transcript from each PASA cluster with the longest ORF')
    parser.add_argument('-if', '--input_pasa_fasta', type=str, required=True, help='Path to PASAs predicted FASTA file' )
    parser.add_argument('-itsc', '--input_pasa_transcript_size_cutoff', type=str, required=False, default=300, help='Nucleotide size cutoff of input transcripts' )
    parser.add_argument('-itc', '--input_pasa_tpm_cutoff', type=str, required=False, default=0.1, help='TPM cutoff of input transcripts' )
    parser.add_argument('-ig', '--input_pasa_gtf', type=str, required=True, help='Path to PASAs predicted GFT file' )
    parser.add_argument('-it', '--input_transdecoder', type=str, required=True, help='Path to the longest_orfs.cds from transdecoder' )
    parser.add_argument('-is', '--input_salmon', type=str, required=True, help='Path to quant.sf file from Salmon' )
    parser.add_argument('-ol', '--output_file_longest_orf', type=str, required=True, help='Path to an output file to be created' )
    parser.add_argument('-oa', '--output_file_most_abundant', type=str, required=True, help='Path to an output file to be created' )
    args = parser.parse_args()

    seqs = utils.fasta_dict_from_file(args.input_pasa_fasta)
    pasa_clusters = load_pasa_clusters(args.input_pasa_gtf)
    orf_lengths = longest_orf_lengths(args.input_transdecoder)
    load_abundances(seqs, args.input_salmon)

    pasa_cluster_count = len(pasa_clusters)
    pasa_transcript_count = len(seqs)
    size_filtered_pasa_transcript_count = 0
    size_filtered_pasa_transcript_count_with_cds = 0
    size_filtered_pasa_cluster_count = 0
    tpm_filtered_pasa_transcript_count = 0
    tpm_filtered_pasa_cluster_count = 0

    print("Stats:\n")
    print("Initial PASA cluster count: {0}".format(pasa_cluster_count))
    print("Initial PASA transcript count: {0}".format(pasa_transcript_count))

    seqs, pasa_clusters = apply_transcript_size_cutoff(seqs, pasa_clusters, args.input_pasa_transcript_size_cutoff)
    seqs, pasa_clusters = apply_abundance_filter(seqs, pasa_clusters, args.input_pasa_tpm_cutoff)
    seqs, pasa_clusters = apply_orf_filter(seqs, pasa_clusters, orf_lengths, 100)

def write_transcripts(seqs, filename):
    with open(filename, 'wt') as fh:
        for asmbl_id in seqs:
            fh.write(">{0} {1}\n{2}\n".format(asmbl_id, seqs[asmbl_id]['h'],
                                              utils.wrapped_fasta(''.join(seqs[asmbl_id]['s']))))

def apply_orf_filter(seqs, clusters, orf_lengths, orf_length_cutoff):
    clusters_to_delete = list()

    for cluster_id in clusters:
        asmbl_ids_to_delete = list()

        for asmbl_id in clusters[cluster_id]:
            if asmbl_id not in orf_lengths or orf_lengths[asmbl_id] < orf_length_cutoff:
                del seqs[asmbl_id]
                asmbl_ids_to_delete.append(asmbl_id)

        for id in asmbl_ids_to_delete:
            del clusters[cluster_id][id]

            if len(clusters[cluster_id]) == 0:
                clusters_to_delete.append(cluster_id)

    for id in clusters_to_delete:
        del clusters[id]

    print("After ORF length filtering:")
    print("\tTranscripts: {0}".format(len(seqs)))
    print("\tPASA clusters: {0}".format(len(clusters)))

    write_transcripts(seqs, 'pasa.orf_filtered.fasta')

    return seqs, clusters

def apply_abundance_filter(seqs, clusters, tpm_cutoff):
    clusters_to_delete = list()

    for cluster_id in clusters:
        asmbl_ids_to_delete = list()

        for asmbl_id in clusters[cluster_id]:
            if 'tpm' in seqs[asmbl_id]:
                if seqs[asmbl_id]['tpm'] < tpm_cutoff:
                    del seqs[asmbl_id]
                    asmbl_ids_to_delete.append(asmbl_id)

        for id in asmbl_ids_to_delete:
            del clusters[cluster_id][id]

            if len(clusters[cluster_id]) == 0:
                clusters_to_delete.append(cluster_id)

    for id in clusters_to_delete:
        del clusters[id]

    print("After TPM filtering:")
    print("\tTranscripts: {0}".format(len(seqs)))
    print("\tPASA clusters: {0}".format(len(clusters)))

    write_transcripts(seqs, 'pasa.tpm_filtered.fasta')

    return seqs, clusters

def apply_transcript_size_cutoff(seqs, clusters, size_cutoff):
    clusters_to_delete = list()

    for cluster_id in clusters:
        asmbl_ids_to_delete = list()

        for asmbl_id in clusters[cluster_id]:
            if len(seqs[asmbl_id]['s']) < size_cutoff:
                del seqs[asmbl_id]
                asmbl_ids_to_delete.append(asmbl_id)

        for id in asmbl_ids_to_delete:
            del clusters[cluster_id][id]

            if len(clusters[cluster_id]) == 0:
                clusters_to_delete.append(cluster_id)

    for id in clusters_to_delete:
        del clusters[id]

    print("After size filtering:")
    print("\tTranscripts: {0}".format(len(seqs)))
    print("\tPASA clusters: {0}".format(len(clusters)))

    write_transcripts(seqs, 'pasa.size_filtered.fasta')

    return seqs, clusters

def load_abundances(seqs, salmon_file):
    line_num = 0
    for line in open(salmon_file):
        line_num += 1
        if line_num == 1:
            continue

        line = line.rstrip()
        cols = line.split("\t")

        if len(cols) < 5:
            continue

        seq_id = cols[0]
        tpm = float(cols[3])

        if seq_id not in seqs:
            raise Exception("ERROR: Got seq ID ({0}) in salmon input that wasn't in FASTA input".format(seq_id))

        seqs[seq_id]['tpm'] = tpm

def load_pasa_clusters(pasa_gtf):
    clusters = dict()
    for line in open(pasa_gtf):
        line = line.rstrip()
        cols = line.split("\t")

        if len(cols) < 9:
            continue

        if cols[2] != 'transcript':
            continue

        m = re.match("gene_id \"(.+?)\"; transcript_id \"align_id:\d+\|(.+?)\";", cols[8])

        if m:
            cluster_id = m.group(1)
            asmbl_id = m.group(2)

            if cluster_id not in clusters:
                clusters[cluster_id] = dict()

            # initialize all their longest-ORF lengths to zero
            clusters[cluster_id][asmbl_id] = 0

    return clusters

def longest_orf_lengths(transdecoder_file):
    orf_lengths = dict()

    for line in open (transdecoder_file):
        line = line.rstrip()
        m = re.match(">.+? type:\S+ len:(\d+) (asmbl_\d+)\:.+", line)
        if m:
            orf_length = int(m.group(1))
            asmbl_id = m.group(2)

            if asmbl_id in orf_lengths:
                if orf_length > orf_lengths[asmbl_id]:
                    orf_lengths[asmbl_id] = orf_length
            else:
                orf_lengths[asmbl_id] = orf_length

    return orf_lengths

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from libdotfiles import HOME_DIR, PKG_DIR, packages, util

packages.try_install("xorg")  # the server itself
packages.try_install("xclip")  # for clip to work
packages.try_install("xorg-xinit")  # for startx
packages.try_install("xorg-xsetroot")  # to fix the mouse cursor
packages.try_install("xorg-xrandr")  # to query monitor information
packages.try_install("xdotool")  # for all sort of things
packages.try_install("autocutsel")  # sync primary and selection clipboards
packages.try_install("clipit")  # keep clipboard data after process exit
packages.try_install("pkg-config")  # for compton
packages.try_install("compton")  # for shadows, transparency and vsync
packages.try_install("shot-git")  # for screenshots

util.create_symlinks(
    [
        (PKG_DIR / "xinitrc", HOME_DIR / ".xinitrc"),
        (PKG_DIR / "compton.conf", HOME_DIR / ".config" / "compton.conf"),
    ]
)
<commit_msg>cfg/xorg: add missing dependency<commit_after>from libdotfiles import HOME_DIR, PKG_DIR, packages, util

packages.try_install("xorg")  # the server itself
packages.try_install("xclip")  # for clip to work
packages.try_install("xorg-xinit")  # for startx
packages.try_install("xorg-xsetroot")  # to fix the mouse cursor
packages.try_install("xorg-xrandr")  # to query monitor information
packages.try_install("xdotool")  # for all sort of things
packages.try_install("autocutsel")  # sync primary and selection clipboards
packages.try_install("clipit")  # keep clipboard data after process exit
packages.try_install("pkg-config")  # for compton
packages.try_install("compton")  # for shadows, transparency and vsync
packages.try_install("shot-git")  # for screenshots
packages.try_install("xdg", method="pip")  # for XDG_CONFIG_HOME

util.create_symlinks(
    [
        (PKG_DIR / "xinitrc", HOME_DIR / ".xinitrc"),
        (PKG_DIR / "compton.conf", HOME_DIR / ".config" / "compton.conf"),
    ]
)
<|endoftext|>


<commit_before>import os
import tarfile

import pytest

from atomic_reactor.constants import (EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE,
                                      IMAGE_TYPE_DOCKER_ARCHIVE)
from atomic_reactor.core import DockerTasker
from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PostBuildPluginsRunner
from atomic_reactor.plugins.post_compress import CompressPlugin
from atomic_reactor.util import ImageName

from tests.constants import INPUT_IMAGE, MOCK

try:
    from six import integer_types
except ImportError:
    integer_types = (int, long)

if MOCK:
    from tests.docker_mock import mock_docker

class Y(object):
    dockerfile_path = None
    path = None

class X(object):
    image_id = INPUT_IMAGE
    source = Y()
    base_image = ImageName.parse('asd')

class TestCompress(object):
    @pytest.mark.parametrize('method, load_exported_image, give_export, extension', [
        ('gzip', False, True, 'gz'),
        ('lzma', False, False, 'xz'),
        ('gzip', True, True, 'gz'),
        ('gzip', True, False, 'gz'),
        ('spam', True, True, None),
    ])
    def test_compress(self, tmpdir, caplog, method, load_exported_image, give_export, extension):
        if MOCK:
            mock_docker()

        tasker = DockerTasker()
        workflow = DockerBuildWorkflow({'provider': 'git', 'uri': 'asd'}, 'test-image')
        workflow.builder = X()
        exp_img = os.path.join(str(tmpdir), 'img.tar')

        if load_exported_image and give_export:
            tarfile.open(exp_img, mode='w').close()
            workflow.exported_image_sequence.append({'path': exp_img,
                                                     'type': IMAGE_TYPE_DOCKER_ARCHIVE})
            tasker = None  # image provided, should not query docker

        runner = PostBuildPluginsRunner(
            tasker,
            workflow,
            [{
                'name': CompressPlugin.key,
                'args': {
                    'method': method,
                    'load_exported_image': load_exported_image,
                },
            }]
        )

        if not extension:
            with pytest.raises(Exception) as excinfo:
                runner.run()
            assert 'Unsupported compression format' in str(excinfo.value)
            return

        runner.run()

        compressed_img = os.path.join(
            workflow.source.tmpdir,
            EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE.format(extension))
        assert os.path.exists(compressed_img)
        metadata = workflow.exported_image_sequence[-1]
        assert metadata['path'] == compressed_img
        assert metadata['type'] == IMAGE_TYPE_DOCKER_ARCHIVE
        assert 'uncompressed_size' in metadata
        assert isinstance(metadata['uncompressed_size'], integer_types)
        assert ", ratio: " in caplog.text()
<commit_msg>py2/3: remove usage of long built-in<commit_after>import os
import tarfile

import pytest

from atomic_reactor.constants import (EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE,
                                      IMAGE_TYPE_DOCKER_ARCHIVE)
from atomic_reactor.core import DockerTasker
from atomic_reactor.inner import DockerBuildWorkflow
from atomic_reactor.plugin import PostBuildPluginsRunner
from atomic_reactor.plugins.post_compress import CompressPlugin
from atomic_reactor.util import ImageName

from tests.constants import INPUT_IMAGE, MOCK

from six import integer_types

if MOCK:
    from tests.docker_mock import mock_docker

class Y(object):
    dockerfile_path = None
    path = None

class X(object):
    image_id = INPUT_IMAGE
    source = Y()
    base_image = ImageName.parse('asd')

class TestCompress(object):
    @pytest.mark.parametrize('method, load_exported_image, give_export, extension', [
        ('gzip', False, True, 'gz'),
        ('lzma', False, False, 'xz'),
        ('gzip', True, True, 'gz'),
        ('gzip', True, False, 'gz'),
        ('spam', True, True, None),
    ])
    def test_compress(self, tmpdir, caplog, method, load_exported_image, give_export, extension):
        if MOCK:
            mock_docker()

        tasker = DockerTasker()
        workflow = DockerBuildWorkflow({'provider': 'git', 'uri': 'asd'}, 'test-image')
        workflow.builder = X()
        exp_img = os.path.join(str(tmpdir), 'img.tar')

        if load_exported_image and give_export:
            tarfile.open(exp_img, mode='w').close()
            workflow.exported_image_sequence.append({'path': exp_img,
                                                     'type': IMAGE_TYPE_DOCKER_ARCHIVE})
            tasker = None  # image provided, should not query docker

        runner = PostBuildPluginsRunner(
            tasker,
            workflow,
            [{
                'name': CompressPlugin.key,
                'args': {
                    'method': method,
                    'load_exported_image': load_exported_image,
                },
            }]
        )

        if not extension:
            with pytest.raises(Exception) as excinfo:
                runner.run()
            assert 'Unsupported compression format' in str(excinfo.value)
            return

        runner.run()

        compressed_img = os.path.join(
            workflow.source.tmpdir,
            EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE.format(extension))
        assert os.path.exists(compressed_img)
        metadata = workflow.exported_image_sequence[-1]
        assert metadata['path'] == compressed_img
        assert metadata['type'] == IMAGE_TYPE_DOCKER_ARCHIVE
        assert 'uncompressed_size' in metadata
        assert isinstance(metadata['uncompressed_size'], integer_types)
        assert ", ratio: " in caplog.text()
<|endoftext|>


<commit_before>import tensorflow as tf

tf.set_random_seed(777)  # reproducibility

def conv2d(x, f=64, k=4, d=2, pad='SAME', name='conv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param d: strides, default 2
    :param pad: padding (valid or same), default same
    :param name: scope name, default conv2d
    :return: conv2d net
    """
    return tf.layers.conv2d(x,
                            filters=f, kernel_size=k, strides=d,
                            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                            kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                            bias_initializer=tf.zeros_initializer(),
                            padding=pad, name=name)

def deconv2d(x, f=64, k=4, d=2, pad='SAME', name='deconv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param d: strides, default 2
    :param pad: padding (valid or same), default same
    :param name: scope name, default deconv2d
    :return: deconv2d net
    """
    return tf.layers.conv2d_transpose(x,
                                      filters=f, kernel_size=k, strides=d,
                                      kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                                      kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                                      bias_initializer=tf.zeros_initializer(),
                                      padding=pad, name=name)

def batch_norm(x, momentum=0.9, eps=1e-5):
    return tf.layers.batch_normalization(inputs=x,
                                         momentum=momentum,
                                         epsilon=eps,
                                         scale=True,
                                         training=True)

class InfoGAN:

    def __init__(self, s, batch_size=64, input_height=28, input_width=28, input_channel=1,
                 sample_num=10 * 10, sample_size=10, output_height=28, output_width=28,
                 df_dim=64, gf_dim=64, fc_unit=1024,
                 n_categories=10, n_continous_factor=2, z_dim=62,
                 g_lr=1e-3, d_lr=2e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param input_height: input image height, default 28
        :param input_width: input image width, default 28
        :param input_channel: input image channel, default 1 (RGB)
        - in case of MNIST, image size is 28x28x1(HWC).

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10
        :param output_height: output images height, default 28
        :param output_width: output images width, default 28

        # Hyper-parameters
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64
        :param fc_unit: fully connected unit, default 1024

        # Training Option
        :param n_categories: the number of categories, default 10 (For MNIST)
        :param n_continous_factor: the number of cont factors, default 2 (For MNIST)
        :param z_dim: z dimension (kinda noise), default 62 (For MNIST)
        :param g_lr: generator learning rate, default 1e-3
        :param d_lr: discriminator learning rate, default 2e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel
        self.image_shape = [self.batch_size, self.input_height, self.input_width, self.input_channel]

        self.sample_num = sample_num
        self.sample_size = sample_size
        self.output_height = output_height
        self.output_width = output_width

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        """
        - MNIST
        n_cat : 10, n_cont : 2, z : 62 => embeddings : 10 + 2 + 62 = 74
        - SVHN
        n_cat : 40, n_cont : 4, z : 124 => embeddings : 40 + 4 + 124 = 168
        - Celeb-A
        n_cat : 100, n_cont : 0, z : 128 => embeddings : 100 + 0 + 128 = 228
        """
        self.n_cat = n_categories         # category dist, label
        self.n_cont = n_continous_factor  # gaussian dist, rotate, etc
        self.z_dim = z_dim
        self.lambda_ = 1  # sufficient for discrete latent codes # less than 1

        self.beta1 = 0.5
        self.beta2 = 0.999
        self.d_lr = d_lr
        self.g_lr = g_lr

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.

        self.g_loss = 0.
        self.d_loss = 0.
        self.q_loss = 0.

        self.d_op = None
        self.g_op = None
        self.q_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.input_height, self.input_width, self.input_channel],
                                name="x-image")                                                # (-1, 32, 32, 3)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name='z-noise')          # (-1, 128)
        self.c = tf.placeholder(tf.float32, shape=[None, self.n_cat + self.n_cont], name='c')  # (-1, 12)

        self.build_infogan()  # build InfoGAN model

    def classifier(self, x, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param x: ~ D
        :param reuse: re-usable
        :return: prob, logits
        """
        with tf.variable_scope("classifier", reuse=reuse):
            x = tf.layers.dense(x, units=128, name='d-fc-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            logits = tf.layers.dense(x, units=self.n_cat + self.n_cont, name='d-fc-2')
            prob = tf.nn.softmax(logits)

            return prob, logits

    def discriminator(self, x, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param x: 28x28x1 images
        :param reuse: re-usable
        :return: logits
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = conv2d(x, f=self.df_dim, name='d-conv2d-0')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = conv2d(x, f=self.df_dim * 2, name='d-conv2d-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            x = tf.layers.flatten(x)

            x = tf.layers.dense(x, units=self.fc_unit, name='d-fc-0')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            logits = tf.layers.dense(x, units=1, name='d-fc-1')
            prob = tf.nn.sigmoid(logits)

            return prob, logits, x

    def generator(self, z, c, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param z: 62 z-noise
        :param c: 10 categories + 2 dimensions
        :param reuse: re-usable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            x = tf.concat([z, c], axis=1)  # (-1, 74)

            x = tf.layers.dense(x, units=self.fc_unit, name='g-fc-0')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            x = tf.layers.dense(x, units=7 * 7 * self.gf_dim * 2, name='g-fc-1')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            x = tf.reshape(x, shape=[-1, 7, 7, self.gf_dim * 2])

            x = deconv2d(x, f=self.gf_dim, name='g-conv2d-0')
            x = tf.nn.leaky_relu(x, alpha=0.1)
            x = batch_norm(x)

            x = deconv2d(x, f=1, name='g-conv2d-2')
            x = tf.nn.sigmoid(x)  # x = tf.nn.tanh(x)

            return x

    def build_infogan(self):
        # Generator
        self.g = self.generator(self.z, self.c)

        # Discriminator
        d_real, d_real_logits, _ = self.discriminator(self.x)
        d_fake, d_fake_logits, d_fake_d = self.discriminator(self.g, reuse=True)

        # Classifier
        c_fake, c_fake_logits = self.classifier(d_fake_d)  # Q net

        # Losses
        d_real_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real_logits,
                                                                             labels=tf.ones_like(d_real)))
        d_fake_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake_logits,
                                                                             labels=tf.zeros_like(d_fake)))
        self.d_loss = d_real_loss + d_fake_loss
        self.g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake_logits,
                                                                             labels=tf.ones_like(d_fake)))

        # categorical
        q_cat_logits = c_fake_logits[:, :self.n_cat]
        q_cat_labels = self.c[:, :self.n_cat]

        q_cat_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=q_cat_logits,
                                                                            labels=q_cat_labels))
        # gaussian
        q_cont_logits = c_fake[:, self.n_cat:]
        q_cont_labels = self.c[:, self.n_cat:]

        q_cont_loss = tf.reduce_mean(tf.reduce_sum(tf.square(q_cont_labels - q_cont_logits), axis=1))  # l2 loss
        self.q_loss = q_cat_loss + q_cont_loss

        # Summary
        tf.summary.histogram("z-noise", self.z)

        # tf.summary.image("g", self.g)  # generated images by Generative Model
        tf.summary.scalar("d_loss", self.d_loss)
        tf.summary.scalar("d_real_loss", d_real_loss)
        tf.summary.scalar("d_fake_loss", d_fake_loss)
        tf.summary.scalar("g_loss", self.g_loss)
        tf.summary.scalar("q_loss", self.q_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]
        q_params = [v for v in t_vars if v.name.startswith('d') or v.name.startswith('g') or v.name.startswith('c')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)
        self.q_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.q_loss, var_list=q_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<commit_msg>fix model<commit_after>import tensorflow as tf

tf.set_random_seed(777)  # reproducibility

def conv2d(x, f=64, k=4, d=2, pad='SAME', name='conv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param d: strides, default 2
    :param pad: padding (valid or same), default same
    :param name: scope name, default conv2d
    :return: conv2d net
    """
    return tf.layers.conv2d(x,
                            filters=f, kernel_size=k, strides=d,
                            kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                            kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                            bias_initializer=tf.zeros_initializer(),
                            padding=pad, name=name)

def deconv2d(x, f=64, k=4, d=2, pad='SAME', name='deconv2d'):
    """
    :param x: input
    :param f: filters, default 64
    :param k: kernel size, default 3
    :param d: strides, default 2
    :param pad: padding (valid or same), default same
    :param name: scope name, default deconv2d
    :return: deconv2d net
    """
    return tf.layers.conv2d_transpose(x,
                                      filters=f, kernel_size=k, strides=d,
                                      kernel_initializer=tf.contrib.layers.variance_scaling_initializer(),
                                      kernel_regularizer=tf.contrib.layers.l2_regularizer(5e-4),
                                      bias_initializer=tf.zeros_initializer(),
                                      padding=pad, name=name)

def batch_norm(x, momentum=0.9, eps=1e-5):
    return tf.layers.batch_normalization(inputs=x,
                                         momentum=momentum,
                                         epsilon=eps,
                                         scale=True,
                                         training=True)

class InfoGAN:

    def __init__(self, s, batch_size=64, input_height=28, input_width=28, input_channel=1,
                 sample_num=10 * 10, sample_size=10, output_height=28, output_width=28,
                 df_dim=64, gf_dim=64, fc_unit=1024,
                 n_categories=10, n_continous_factor=2, z_dim=62,
                 g_lr=1e-3, d_lr=2e-4):

        """
        # General Settings
        :param s: TF Session
        :param batch_size: training batch size, default 64
        :param input_height: input image height, default 28
        :param input_width: input image width, default 28
        :param input_channel: input image channel, default 1 (RGB)
        - in case of MNIST, image size is 28x28x1(HWC).

        # Output Settings
        :param sample_num: the number of output images, default 100
        :param sample_size: sample image size, default 10
        :param output_height: output images height, default 28
        :param output_width: output images width, default 28

        # Hyper-parameters
        :param df_dim: discriminator filter, default 64
        :param gf_dim: generator filter, default 64
        :param fc_unit: fully connected unit, default 1024

        # Training Option
        :param n_categories: the number of categories, default 10 (For MNIST)
        :param n_continous_factor: the number of cont factors, default 2 (For MNIST)
        :param z_dim: z dimension (kinda noise), default 62 (For MNIST)
        :param g_lr: generator learning rate, default 1e-3
        :param d_lr: discriminator learning rate, default 2e-4
        """

        self.s = s
        self.batch_size = batch_size

        self.input_height = input_height
        self.input_width = input_width
        self.input_channel = input_channel
        self.image_shape = [self.batch_size, self.input_height, self.input_width, self.input_channel]

        self.sample_num = sample_num
        self.sample_size = sample_size
        self.output_height = output_height
        self.output_width = output_width

        self.df_dim = df_dim
        self.gf_dim = gf_dim
        self.fc_unit = fc_unit

        """
        - MNIST
        n_cat : 10, n_cont : 2, z : 62 => embeddings : 10 + 2 + 62 = 74
        - SVHN
        n_cat : 40, n_cont : 4, z : 124 => embeddings : 40 + 4 + 124 = 168
        - Celeb-A
        n_cat : 100, n_cont : 0, z : 128 => embeddings : 100 + 0 + 128 = 228
        """
        self.n_cat = n_categories         # category dist, label
        self.n_cont = n_continous_factor  # gaussian dist, rotate, etc
        self.z_dim = z_dim
        self.lambda_ = 1  # sufficient for discrete latent codes # less than 1

        self.beta1 = 0.5
        self.beta2 = 0.999
        self.d_lr = d_lr
        self.g_lr = g_lr

        # pre-defined
        self.d_real = 0.
        self.d_fake = 0.

        self.g_loss = 0.
        self.d_loss = 0.
        self.q_loss = 0.

        self.d_op = None
        self.g_op = None
        self.q_op = None

        self.merged = None
        self.writer = None
        self.saver = None

        # Placeholders
        self.x = tf.placeholder(tf.float32,
                                shape=[None, self.input_height, self.input_width, self.input_channel],
                                name="x-image")                                                # (-1, 32, 32, 3)
        self.z = tf.placeholder(tf.float32, shape=[None, self.z_dim], name='z-noise')          # (-1, 128)
        self.c = tf.placeholder(tf.float32, shape=[None, self.n_cat + self.n_cont], name='c')  # (-1, 12)

        self.build_infogan()  # build InfoGAN model

    def classifier(self, x, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param x: ~ D
        :param reuse: re-usable
        :return: prob, logits
        """
        with tf.variable_scope("classifier", reuse=reuse):
            x = tf.layers.dense(x, units=128, name='d-fc-1')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            logits = tf.layers.dense(x, units=self.n_cat + self.n_cont, name='d-fc-2')
            prob = tf.nn.softmax(logits)

            return prob, logits

    def discriminator(self, x, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param x: 28x28x1 images
        :param reuse: re-usable
        :return: logits
        """
        with tf.variable_scope("discriminator", reuse=reuse):
            x = conv2d(x, f=self.df_dim, name='d-conv2d-0')
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = conv2d(x, f=self.df_dim * 2, name='d-conv2d-1')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = tf.layers.flatten(x)

            x = tf.layers.dense(x, units=self.fc_unit, name='d-fc-0')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            logits = tf.layers.dense(x, units=1, name='d-fc-1')
            prob = tf.nn.sigmoid(logits)

            return prob, logits, x

    def generator(self, z, c, reuse=None):
        """
        # This is a network architecture for MNIST DataSet referred in the paper
        :param z: 62 z-noise
        :param c: 10 categories + 2 dimensions
        :param reuse: re-usable
        :return: prob
        """
        with tf.variable_scope("generator", reuse=reuse):
            x = tf.concat([z, c], axis=1)  # (-1, 74)

            x = tf.layers.dense(x, units=self.fc_unit, name='g-fc-0')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = tf.layers.dense(x, units=7 * 7 * self.gf_dim * 2, name='g-fc-1')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = tf.reshape(x, shape=[-1, 7, 7, self.gf_dim * 2])

            x = deconv2d(x, f=self.gf_dim, name='g-conv2d-0')
            x = batch_norm(x)
            x = tf.nn.leaky_relu(x, alpha=0.1)

            x = deconv2d(x, f=1, name='g-conv2d-2')
            x = tf.nn.sigmoid(x)  # x = tf.nn.tanh(x)

            return x

    def build_infogan(self):
        # Generator
        self.g = self.generator(self.z, self.c)

        # Discriminator
        d_real, d_real_logits, _ = self.discriminator(self.x)
        d_fake, d_fake_logits, d_fake_d = self.discriminator(self.g, reuse=True)

        # Classifier
        c_fake, c_fake_logits = self.classifier(d_fake_d)  # Q net

        # Losses
        d_real_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_real_logits,
                                                                             labels=tf.ones_like(d_real)))
        d_fake_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake_logits,
                                                                             labels=tf.zeros_like(d_fake)))
        self.d_loss = d_real_loss + d_fake_loss
        self.g_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=d_fake_logits,
                                                                             labels=tf.ones_like(d_fake)))

        # categorical
        q_cat_logits = c_fake_logits[:, :self.n_cat]
        q_cat_labels = self.c[:, :self.n_cat]

        q_cat_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=q_cat_logits,
                                                                            labels=q_cat_labels))
        # gaussian
        q_cont_logits = c_fake[:, self.n_cat:]
        q_cont_labels = self.c[:, self.n_cat:]

        q_cont_loss = tf.reduce_mean(tf.reduce_sum(tf.square(q_cont_labels - q_cont_logits), axis=1))  # l2 loss
        self.q_loss = q_cat_loss + q_cont_loss

        # Summary
        tf.summary.histogram("z-noise", self.z)

        # tf.summary.image("g", self.g)  # generated images by Generative Model
        tf.summary.scalar("d_loss", self.d_loss)
        tf.summary.scalar("d_real_loss", d_real_loss)
        tf.summary.scalar("d_fake_loss", d_fake_loss)
        tf.summary.scalar("g_loss", self.g_loss)
        tf.summary.scalar("q_loss", self.q_loss)

        # Optimizer
        t_vars = tf.trainable_variables()
        d_params = [v for v in t_vars if v.name.startswith('d')]
        g_params = [v for v in t_vars if v.name.startswith('g')]
        q_params = [v for v in t_vars if v.name.startswith('d') or v.name.startswith('g') or v.name.startswith('c')]

        self.d_op = tf.train.AdamOptimizer(learning_rate=self.d_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.d_loss, var_list=d_params)
        self.g_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.g_loss, var_list=g_params)
        self.q_op = tf.train.AdamOptimizer(learning_rate=self.g_lr,
                                           beta1=self.beta1, beta2=self.beta2).minimize(self.q_loss, var_list=q_params)

        # Merge summary
        self.merged = tf.summary.merge_all()

        # Model saver
        self.saver = tf.train.Saver(max_to_keep=1)
        self.writer = tf.summary.FileWriter('./model/', self.s.graph)
<|endoftext|>


<commit_before>'''
Unique version information place
'''
__version__ = '0.2.0'
VERSION = tuple(int(x) for x in __version__.split('.'))
<commit_msg>0.2.1 release<commit_after>'''
Unique version information place
'''
__version__ = '0.2.1'
VERSION = tuple(int(x) for x in __version__.split('.'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Map tags generated by inriaxmlwrapper to a better set of names

@author: Karthik Madathil(@kmadathil)

'''

from __future__ import print_function
from  sanskrit_parser.base.SanskritBase import SanskritObject, SLP1, SCHEMES

inriatags= [ ('v-cj-prim', 'प्राथमिकः'),
    ('v-cj-ca', 'णिजन्तः'),
    ('v-cj-int', 'यङन्तः'),
    ('v-cj-des', 'सन्नन्तः'),
    ('sys-prs-md-pr', 'लट्'),
    ('sys-prs-md-ip', 'लोट्'),
    ('sys-prs-md-op', 'विधिलिङ्'),
    ('sys-prs-md-im', 'लङ्'),
    ('sys-pas-md-pr', 'लट्-कर्मणि'),
    ('sys-pas-md-ip', 'लोट्-कर्मणि'),
    ('sys-pas-md-op', 'विधिलिङ्-कर्मणि'),
    ('sys-pas-md-im', 'लङ्-कर्मणि'),
    ('sys-tp-fut', 'लृट्'),
    ('sys-tp-prf', 'लिट्'),
    ('sys-tp-aor', 'लुङ्'),
    ('sys-tp-inj', 'आगमाभावयुक्तलुङ्'),
    ('sys-tp-cnd', 'लृङ्'),
    ('sys-tp-ben', 'आशीर्लिङ्'),
    ('sys-pef', 'लुट्'),
    ('np-sg', 'एकवचनम्'),
    ('np-du', 'द्विवचनम्'),
    ('np-pl', 'बहुवचनम्'),
    ('fst', 'उत्तमपुरुषः'),
    ('snd', 'मध्यमपुरुषः'),
    ('trd', 'प्रथमपुरुषः'),
    ('na-nom', 'प्रथमाविभक्तिः'),
    ('na-voc', 'संबोधनविभक्तिः'),
    ('na-acc', 'द्वितीयाविभक्तिः'),
    ('na-ins', 'तृतीयाविभक्तिः'),
    ('na-dat', 'चतुर्थीविभक्तिः'),
    ('na-abl', 'पञ्चमीविभक्तिः'),
    ('na-gen', 'षष्ठीविभक्तिः'),
    ('na-loc', 'सप्तमीविभक्तिः'),
    ('sg', 'एकवचनम्'),
    ('du', 'द्विवचनम्'),
    ('pl', 'बहुवचनम्'),
    ('mas', 'पुंल्लिङ्गम्'),
    ('fem', 'स्त्रीलिङ्गम्'),
    ('neu', 'नपुंसकलिङ्गम्'),
    ('dei', 'सङ्ख्या'),
    ('uf', 'अव्ययम्'),
    ('ind', 'क्रियाविशेषणम्'),
    ('interj', 'उद्गारः'),
    ('parti', 'निपातम्'),
    ('prep', 'चादिः'),
    ('conj', 'संयोजकः'),
    ('tasil', 'तसिल्'),
    ('vu-cj-prim', 'अव्ययधातुरूप-प्राथमिकः'),
    ('vu-cj-ca', 'अव्ययधातुरूप-णिजन्तः'),
    ('vu-cj-int', 'अव्ययधातुरूप-यङन्तः'),
    ('vu-cj-des', 'अव्ययधातुरूप-सन्नन्तः'),
    ('iv-inf','तुमुन्'),
    ('iv-abs','क्त्वा'),
    ('iv-per','per'),
    ('ab-cj-prim', 'क्त्वा-प्राथमिकः'),
    ('ab-cj-ca', 'क्त्वा-णिजन्तः'),
    ('ab-cj-int', 'क्त्वा-यङन्तः'),
    ('ab-cj-des', 'क्त्वा-सन्नन्तः'),
    ('kr-cj-prim-no', 'प्राथमिकः'),
    ('kr-cj-ca-no', 'णिजन्तः'),
    ('kr-cj-int-no', 'यङन्तः'),
    ('kr-cj-des-no', 'सन्नन्तः'),
    ('kr-vb-no', ''),
    ('ppp', 'कर्मणिभूतकृदन्तः'),
    ('ppa', 'कर्तरिभूतकृदन्तः'),
    ('pprp', 'कर्मणिवर्तमानकृदन्तः'),
    ('ppr-para', 'कर्तरिवर्तमानकृदन्त-परस्मैपदी'),
    ('ppr-atma', 'कर्तरिवर्तमानकृदन्त-आत्मनेपदी'),
    ('ppft-para', 'पूर्णभूतकृदन्त-परस्मैपदी'),
    ('ppft-atma', 'पूर्णभूतकृदन्त-आत्मनेपदी'),
    ('pfutp', 'कर्मणिभविष्यत्कृदन्तः'),
    ('pfut-para', 'कर्तरिभविष्यत्कृदन्त-परस्मैपदी'),
    ('pfut-atma', 'कर्तरिभविष्यत्कृदन्त-आत्मनेपदी'),
    ('gya', 'य'),
    ('iya', 'ईय'),
    ('tav', 'तव्य'),
    ('para', 'कर्तरि'),
    ('atma', 'कर्तरि'),
    ('pass', 'कर्मणि'),
    ('pa', 'कृदन्तः'),
    ('iic', 'समासपूर्वपदनामपदम्'),
    ('iip', 'समासपूर्वपदकृदन्तः'),
    ('iiv', 'समासपूर्वपदधातुः'),
    ('upsrg', 'उपसर्गः')]

def _inriaTagsToDb(tag):
    # ('tag','ourtag)
    itag=tag[0]
    otag=tag[1]
    if itag.find('-') != -1:
        iset=set(itag.split('-'))
    else:
        iset=set([itag])
    return (iset,SanskritObject(otag,encoding=SCHEMES['Devanagari']))

inriatagdb = map(_inriaTagsToDb,inriatags)

def inriaMapTag(tag):
    ''' Map an INRIA tag to our format

    Params:
       tag(tuple) : (<stem>,set([inria tags]))
    Returns
       tuple : (<stem>,set([our tags]))
    '''
    stem=tag[0]
    tset=tag[1]
    olist = []
    for s in inriatagdb:
        if s[0] <= tset:
            olist.append(s[1])
    return (stem,set(olist))

def inriaTagMapper(tags):
    ''' Map an INRIA tag to our format

    Params:
       tags: List of INRIA format tags
    Returns
       list[(<stem>,set([our tags])) ...]
    '''
    return map(inriaMapTag,tags)

if __name__ == "__main__":
    import sys
    if len(sys.argv)>=2:
        print(map(lambda x: inriaMapTag(eval(x)),sys.argv[1:]))
    else:
        print("Need at least one argument to convert")
<commit_msg>fixed double vacana in tags<commit_after># -*- coding: utf-8 -*-
'''
Map tags generated by inriaxmlwrapper to a better set of names

@author: Karthik Madathil(@kmadathil)

'''

from __future__ import print_function
from  sanskrit_parser.base.SanskritBase import SanskritObject, SLP1, SCHEMES

inriatags= [ ('v-cj-prim', 'प्राथमिकः'),
    ('v-cj-ca', 'णिजन्तः'),
    ('v-cj-int', 'यङन्तः'),
    ('v-cj-des', 'सन्नन्तः'),
    ('sys-prs-md-pr', 'लट्'),
    ('sys-prs-md-ip', 'लोट्'),
    ('sys-prs-md-op', 'विधिलिङ्'),
    ('sys-prs-md-im', 'लङ्'),
    ('sys-pas-md-pr', 'लट्-कर्मणि'),
    ('sys-pas-md-ip', 'लोट्-कर्मणि'),
    ('sys-pas-md-op', 'विधिलिङ्-कर्मणि'),
    ('sys-pas-md-im', 'लङ्-कर्मणि'),
    ('sys-tp-fut', 'लृट्'),
    ('sys-tp-prf', 'लिट्'),
    ('sys-tp-aor', 'लुङ्'),
    ('sys-tp-inj', 'आगमाभावयुक्तलुङ्'),
    ('sys-tp-cnd', 'लृङ्'),
    ('sys-tp-ben', 'आशीर्लिङ्'),
    ('sys-pef', 'लुट्'),
#    ('np-sg', 'एकवचनम्'),
#    ('np-du', 'द्विवचनम्'),
#    ('np-pl', 'बहुवचनम्'),
    ('fst', 'उत्तमपुरुषः'),
    ('snd', 'मध्यमपुरुषः'),
    ('trd', 'प्रथमपुरुषः'),
    ('na-nom', 'प्रथमाविभक्तिः'),
    ('na-voc', 'संबोधनविभक्तिः'),
    ('na-acc', 'द्वितीयाविभक्तिः'),
    ('na-ins', 'तृतीयाविभक्तिः'),
    ('na-dat', 'चतुर्थीविभक्तिः'),
    ('na-abl', 'पञ्चमीविभक्तिः'),
    ('na-gen', 'षष्ठीविभक्तिः'),
    ('na-loc', 'सप्तमीविभक्तिः'),
    ('sg', 'एकवचनम्'),
    ('du', 'द्विवचनम्'),
    ('pl', 'बहुवचनम्'),
    ('mas', 'पुंल्लिङ्गम्'),
    ('fem', 'स्त्रीलिङ्गम्'),
    ('neu', 'नपुंसकलिङ्गम्'),
    ('dei', 'सङ्ख्या'),
    ('uf', 'अव्ययम्'),
    ('ind', 'क्रियाविशेषणम्'),
    ('interj', 'उद्गारः'),
    ('parti', 'निपातम्'),
    ('prep', 'चादिः'),
    ('conj', 'संयोजकः'),
    ('tasil', 'तसिल्'),
    ('vu-cj-prim', 'अव्ययधातुरूप-प्राथमिकः'),
    ('vu-cj-ca', 'अव्ययधातुरूप-णिजन्तः'),
    ('vu-cj-int', 'अव्ययधातुरूप-यङन्तः'),
    ('vu-cj-des', 'अव्ययधातुरूप-सन्नन्तः'),
    ('iv-inf','तुमुन्'),
    ('iv-abs','क्त्वा'),
    ('iv-per','per'),
    ('ab-cj-prim', 'क्त्वा-प्राथमिकः'),
    ('ab-cj-ca', 'क्त्वा-णिजन्तः'),
    ('ab-cj-int', 'क्त्वा-यङन्तः'),
    ('ab-cj-des', 'क्त्वा-सन्नन्तः'),
    ('kr-cj-prim-no', 'प्राथमिकः'),
    ('kr-cj-ca-no', 'णिजन्तः'),
    ('kr-cj-int-no', 'यङन्तः'),
    ('kr-cj-des-no', 'सन्नन्तः'),
    ('kr-vb-no', ''),
    ('ppp', 'कर्मणिभूतकृदन्तः'),
    ('ppa', 'कर्तरिभूतकृदन्तः'),
    ('pprp', 'कर्मणिवर्तमानकृदन्तः'),
    ('ppr-para', 'कर्तरिवर्तमानकृदन्त-परस्मैपदी'),
    ('ppr-atma', 'कर्तरिवर्तमानकृदन्त-आत्मनेपदी'),
    ('ppft-para', 'पूर्णभूतकृदन्त-परस्मैपदी'),
    ('ppft-atma', 'पूर्णभूतकृदन्त-आत्मनेपदी'),
    ('pfutp', 'कर्मणिभविष्यत्कृदन्तः'),
    ('pfut-para', 'कर्तरिभविष्यत्कृदन्त-परस्मैपदी'),
    ('pfut-atma', 'कर्तरिभविष्यत्कृदन्त-आत्मनेपदी'),
    ('gya', 'य'),
    ('iya', 'ईय'),
    ('tav', 'तव्य'),
    ('para', 'कर्तरि'),
    ('atma', 'कर्तरि'),
    ('pass', 'कर्मणि'),
    ('pa', 'कृदन्तः'),
    ('iic', 'समासपूर्वपदनामपदम्'),
    ('iip', 'समासपूर्वपदकृदन्तः'),
    ('iiv', 'समासपूर्वपदधातुः'),
    ('upsrg', 'उपसर्गः')]

def _inriaTagsToDb(tag):
    # ('tag','ourtag)
    itag=tag[0]
    otag=tag[1]
    if itag.find('-') != -1:
        iset=set(itag.split('-'))
    else:
        iset=set([itag])
    return (iset,SanskritObject(otag,encoding=SCHEMES['Devanagari']))

inriatagdb = map(_inriaTagsToDb,inriatags)

def inriaMapTag(tag):
    ''' Map an INRIA tag to our format

    Params:
       tag(tuple) : (<stem>,set([inria tags]))
    Returns
       tuple : (<stem>,set([our tags]))
    '''
    stem=tag[0]
    tset=tag[1]
    olist = []
    for s in inriatagdb:
        if s[0] <= tset:
            olist.append(s[1])
    return (stem,set(olist))

def inriaTagMapper(tags):
    ''' Map an INRIA tag to our format

    Params:
       tags: List of INRIA format tags
    Returns
       list[(<stem>,set([our tags])) ...]
    '''
    return map(inriaMapTag,tags)

if __name__ == "__main__":
    import sys
    if len(sys.argv)>=2:
        print(map(lambda x: inriaMapTag(eval(x)),sys.argv[1:]))
    else:
        print("Need at least one argument to convert")
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, unicode_literals

import logging

from collections import namedtuple

from . import connection
from . import protocol

logger = logging.getLogger(__name__)

class ControlUnit(object):
    """Interface to a Carrera Digital 124/132 Control Unit."""

    class Status(namedtuple('Status', 'fuel start mode pit display')):

        __slots__ = ()

        MODE_FUEL = 1
        MODE_REAL = 2
        MODE_PIT_LANE = 4
        MODE_LAP_COUNTER = 8

    class Timer(namedtuple('Timer', 'address timestamp sector')):
        pass

    def __init__(self, device, **kwargs):
        if isinstance(device, connection.Connection):
            self.__connection = device
        else:
            logger.debug('Connecting to %s', device)
            self.__connection = connection.open(device, **kwargs)
            logger.debug('Connection established')

    def close(self):
        """Close the connection to the CU."""
        logger.debug('Closing connection')
        self.__connection.close()

    def clrpos(self):
        """Clear all position information."""
        self.setword(6, 0, 9)

    def ignore(self, mask):
        """Ignore the controllers represented by bitmask `mask`."""
        self.request(protocol.pack('cBC', b':', mask))

    def request(self, buf=b'?', maxlength=None):
        """Send a message to the CU and wait for a response."""
        logger.debug('Sending message %r', buf)
        self.__connection.send(buf)
        while True:
            res = self.__connection.recv(maxlength)
            if res.startswith(buf[0:1]):
                break
            else:
                logger.warn('Received unexpected message %r', res)
        logger.debug('Received message %r', res)
        if res.startswith(b'?:'):
            # recent CU versions report two extra unknown bytes with '?:'
            try:
                parts = protocol.unpack('2x8YYYBYC', res)
            except protocol.ChecksumError:
                parts = protocol.unpack('2x8YYYBYxxC', res)
            fuel, (start, mode, pitmask, display) = parts[:8], parts[8:]
            pit = tuple(pitmask & (1 << n) != 0 for n in range(8))
            return ControlUnit.Status(fuel, start, mode, pit, display)
        elif res.startswith(b'?'):
            address, timestamp, sector = protocol.unpack('xYIYC', res)
            return ControlUnit.Timer(address - 1, timestamp, sector)
        else:
            return res

    def reset(self):
        """Reset the CU timer."""
        self.request(b'=10')

    def setbrake(self, address, value):
        """Set the brake value for controller `address`."""
        self.setword(1, address, value, repeat=2)

    def setfuel(self, address, value):
        """Set the fuel value for controller `address`."""
        self.setword(2, address, value, repeat=2)

    def setlap(self, value):
        """Set the current lap."""
        if value < 0 or value > 255:
            raise ValueError('Lap value out of range')
        self.setlap_hi(value >> 4)
        self.setlap_lo(value & 0xf)

    def setlap_hi(self, value):
        """Set the high nibble of the current lap."""
        self.setword(17, 7, value)

    def setlap_lo(self, value):
        """Set the low nibble of the current lap."""
        self.setword(18, 7, value)

    def setpos(self, address, position):
        """Set the current position for controller address."""
        if position < 1 or position > 8:
            raise ValueError('Position out of range')
        self.setword(6, address, position)

    def setspeed(self, address, value):
        """Set the speed value for controller address."""
        self.setword(0, address, value, repeat=2)

    def setword(self, word, address, value, repeat=1):
        if word < 0 or word > 31:
            raise ValueError('Command word out of range')
        if address < 0 or address > 7:
            raise ValueError('Address out of range')
        if value < 0 or value > 15:
            raise ValueError('Value out of range')
        if repeat < 1 or repeat > 15:
            raise ValueError('Repeat count out of range')
        buf = protocol.pack('cBYYC', b'J', word | address << 5, value, repeat)
        return self.request(buf)

    def start(self):
        """Initiate CU start sequence."""
        self.request(b'T2')

    def version(self):
        """Retrieve CU version."""
        return protocol.unpack('x4sC', self.request(b'0'))[0]
<commit_msg>Keep flake8 happy.<commit_after>from __future__ import absolute_import, division, unicode_literals

import logging
from collections import namedtuple

from . import connection
from . import protocol

logger = logging.getLogger(__name__)

class ControlUnit(object):
    """Interface to a Carrera Digital 124/132 Control Unit."""

    class Status(namedtuple('Status', 'fuel start mode pit display')):

        __slots__ = ()

        MODE_FUEL = 1
        MODE_REAL = 2
        MODE_PIT_LANE = 4
        MODE_LAP_COUNTER = 8

    class Timer(namedtuple('Timer', 'address timestamp sector')):
        pass

    def __init__(self, device, **kwargs):
        if isinstance(device, connection.Connection):
            self.__connection = device
        else:
            logger.debug('Connecting to %s', device)
            self.__connection = connection.open(device, **kwargs)
            logger.debug('Connection established')

    def close(self):
        """Close the connection to the CU."""
        logger.debug('Closing connection')
        self.__connection.close()

    def clrpos(self):
        """Clear all position information."""
        self.setword(6, 0, 9)

    def ignore(self, mask):
        """Ignore the controllers represented by bitmask `mask`."""
        self.request(protocol.pack('cBC', b':', mask))

    def request(self, buf=b'?', maxlength=None):
        """Send a message to the CU and wait for a response."""
        logger.debug('Sending message %r', buf)
        self.__connection.send(buf)
        while True:
            res = self.__connection.recv(maxlength)
            if res.startswith(buf[0:1]):
                break
            else:
                logger.warn('Received unexpected message %r', res)
        logger.debug('Received message %r', res)
        if res.startswith(b'?:'):
            # recent CU versions report two extra unknown bytes with '?:'
            try:
                parts = protocol.unpack('2x8YYYBYC', res)
            except protocol.ChecksumError:
                parts = protocol.unpack('2x8YYYBYxxC', res)
            fuel, (start, mode, pitmask, display) = parts[:8], parts[8:]
            pit = tuple(pitmask & (1 << n) != 0 for n in range(8))
            return ControlUnit.Status(fuel, start, mode, pit, display)
        elif res.startswith(b'?'):
            address, timestamp, sector = protocol.unpack('xYIYC', res)
            return ControlUnit.Timer(address - 1, timestamp, sector)
        else:
            return res

    def reset(self):
        """Reset the CU timer."""
        self.request(b'=10')

    def setbrake(self, address, value):
        """Set the brake value for controller `address`."""
        self.setword(1, address, value, repeat=2)

    def setfuel(self, address, value):
        """Set the fuel value for controller `address`."""
        self.setword(2, address, value, repeat=2)

    def setlap(self, value):
        """Set the current lap."""
        if value < 0 or value > 255:
            raise ValueError('Lap value out of range')
        self.setlap_hi(value >> 4)
        self.setlap_lo(value & 0xf)

    def setlap_hi(self, value):
        """Set the high nibble of the current lap."""
        self.setword(17, 7, value)

    def setlap_lo(self, value):
        """Set the low nibble of the current lap."""
        self.setword(18, 7, value)

    def setpos(self, address, position):
        """Set the current position for controller address."""
        if position < 1 or position > 8:
            raise ValueError('Position out of range')
        self.setword(6, address, position)

    def setspeed(self, address, value):
        """Set the speed value for controller address."""
        self.setword(0, address, value, repeat=2)

    def setword(self, word, address, value, repeat=1):
        if word < 0 or word > 31:
            raise ValueError('Command word out of range')
        if address < 0 or address > 7:
            raise ValueError('Address out of range')
        if value < 0 or value > 15:
            raise ValueError('Value out of range')
        if repeat < 1 or repeat > 15:
            raise ValueError('Repeat count out of range')
        buf = protocol.pack('cBYYC', b'J', word | address << 5, value, repeat)
        return self.request(buf)

    def start(self):
        """Initiate CU start sequence."""
        self.request(b'T2')

    def version(self):
        """Retrieve CU version."""
        return protocol.unpack('x4sC', self.request(b'0'))[0]
<|endoftext|>