<commit_before>import json
import hashlib
import random
import types
import unicodedata
from binascii import hexlify as hexify, unhexlify as unhexify

def hash_fn():
    return hashlib.sha256()

def hash_primitive(t, b):
    #print hexify(t), hexify(b)
    m = hash_fn()
    m.update(t)
    m.update(b)
    t = m.digest()
    #print '=', hexify(t)
    return t

# We need this class because otherwise we can't put a list in a set.
class FrozenList(object):
    def __init__(self, l):
        self.l = tuple(l)

    def __getitem__(self, key):
        return self.l[key]

    def __hash__(self):
        return hash(self.l)

    def __eq__(self, other):
        return self.l == other.l

    def __len__(self):
        return len(self.l)

def obj_hash_bool(b):
    return hash_primitive('b', '1' if b else '0')

def obj_hash_list(l):
    h = ''
    for o in l:
        h += obj_hash(o)
    return hash_primitive('l', h)

def unicode_normalize(u):
    return unicodedata.normalize('NFC', u).encode('utf-8')

def obj_hash_dict(d):
    h = ''
    kh = [obj_hash(k) + obj_hash(v) for (k, v) in d.items()]
    for v in sorted(kh):
        h += v
    return hash_primitive('d', h)

def obj_hash_unicode(u):
    return hash_primitive('u', unicode_normalize(u))

def float_normalize(f):
    # special case 0
    # Note that if we allowed f to end up > .5 or == 0, we'd get the same thing
    if f == 0.0:
        return '+0:'
    # sign
    s = '+'
    if f < 0:
        s = '-'
        f = -f
    # exponent
    e = 0
    while f > 1:
        f /= 2
        e += 1
    while f <= .5:
        f *= 2
        e -= 1
    s += str(e) + ':'
    # mantissa
    assert f <= 1
    assert f > .5
    while f:
        if f >= 1:
            s += '1'
            f -= 1
        else:
            s += '0'
        assert f < 1
        assert len(s) < 1000
        f *= 2

    return s

def obj_hash_float(f):
    return hash_primitive('f', float_normalize(f))

def obj_hash_int(i):
    return hash_primitive('i', str(i))

def obj_hash_set(s):
    h = []
    for e in s:
        h.append(obj_hash(e))
    r = ''
    for t in sorted(h):
        r += t
    return hash_primitive('s', r)

class Redacted(object):
    def __init__(self, hash):
        self.hash = unhexify(hash)

class RedactedObject(Redacted):
    def __init__(self, o):
        self.hash = obj_hash(o)

def obj_hash(o):
    if type(o) is list or type(o) is FrozenList:
        return obj_hash_list(o)
    elif type(o) is dict:
        return obj_hash_dict(o)
    elif type(o) is unicode:
        return obj_hash_unicode(o)
    elif type(o) is float:
        return obj_hash_float(o)
    elif type(o) is int:
        return obj_hash_int(o)
    elif type(o) is str:
        return obj_hash_unicode(unicode(o))
    elif type(o) is set or type(o) is frozenset:
        return obj_hash_set(o)
    elif type(o) is bool:
        return obj_hash_bool(o)
    elif isinstance(o, Redacted):
        return o.hash
    elif o is None:
        return hash_primitive('n', '')

    print type(o)
    assert False

def python_json_hash(j):
    t = json.loads(j)
    return obj_hash(t)

def is_primitive_type(t):
    return t is str or t is unicode or t is float or t is int or t is bool or t is types.NoneType

class ApplyToLeaves(object):
    def __init__(self, leaf_fn, restrict = None):
        self.leaf_fn = leaf_fn
        self.restrict = restrict

    def __call__(self, o):
        t = type(o)
        if t is dict:
            return {self(k): self(v) for (k,v) in o.items()}
        elif t is list:
            return [self(e) for e in o]
        elif t is set:
            return set([self(e) for e in o])
        elif is_primitive_type(t):
            if self.restrict:
                for tt in self.restrict:
                    if t is tt:
                        return self.leaf_fn(o)
                return o
            else:
                return self.leaf_fn(o)

        print type(o)
        assert False

commonize = ApplyToLeaves(lambda o: float(o), (int,))

def common_json_hash(j):
    t = json.loads(j)
    t = commonize(t)
    return obj_hash(t)

def redactize_unicode(u):
    if u.startswith('**REDACTED**'):
        return Redacted(u[12:])
    else:
        return u

redactize = ApplyToLeaves(redactize_unicode, (str, unicode))

def common_redacted_json_hash(j):
    t = json.loads(j)
    t = commonize(t)
    t = redactize(t)
    return obj_hash(t)

class ApplyToLeavesAndKeys(ApplyToLeaves):
    def __init__(self, leaf_fn, key_fn):
        ApplyToLeaves.__init__(self, leaf_fn)
        self.key_fn = key_fn

    def __call__(self, o):
        if type(o) is dict:
            return {self.key_fn(k): self(v) for (k, v) in o.items()}
        return ApplyToLeaves.__call__(self, o)

def redactable_entity(e):
    return FrozenList([redactable_rand(), e])

def redactable_key(k):
    return redactable_rand() + k

def redactable_rand():
    r = ''
    for x in range(32):
        r += chr(random.SystemRandom().getrandbits(8))
    return hexify(r)

redactable = ApplyToLeavesAndKeys(redactable_entity, redactable_key)

def unredactable_dict(d):
    return {unredactable_key(k): unredactable(v) for (k, v) in d.items()}

def unredactable_key(k):
    return k[32:]

def unredactable_list(l):
    return [unredactable(e) for e in l]

def unredactable_set(s):
    return set([unredactable(e) for e in s])

def unredactable(o):
    t = type(o)
    if (t is list or t is FrozenList) and len(o) == 2 and type(o[0]) is str:
        assert is_primitive_type(type(o[1]))
        return o[1]
    elif t is dict:
        return unredactable_dict(o)
    elif t is list:
        return unredactable_list(o)
    elif t is set:
        return unredactable_set(o)

    print type(o)
    assert False

def unicode_normalize_entity(e):
    if type(e) is unicode:
        return unicode_normalize(e)
    elif type(e) is str:
        return unicode_normalize(unicode(e))
    return e

unicode_normalizify = ApplyToLeaves(unicode_normalize_entity)
<commit_msg>D'oh!<commit_after>import json
import hashlib
import random
import types
import unicodedata
from binascii import hexlify as hexify, unhexlify as unhexify

def hash_fn():
    return hashlib.sha256()

def hash_primitive(t, b):
    #print hexify(t), hexify(b)
    m = hash_fn()
    m.update(t)
    m.update(b)
    t = m.digest()
    #print '=', hexify(t)
    return t

# We need this class because otherwise we can't put a list in a set.
class FrozenList(object):
    def __init__(self, l):
        self.l = tuple(l)

    def __getitem__(self, key):
        return self.l[key]

    def __hash__(self):
        return hash(self.l)

    def __eq__(self, other):
        return self.l == other.l

    def __len__(self):
        return len(self.l)

def obj_hash_bool(b):
    return hash_primitive('b', '1' if b else '0')

def obj_hash_list(l):
    h = ''
    for o in l:
        h += obj_hash(o)
    return hash_primitive('l', h)

def unicode_normalize(u):
    return unicodedata.normalize('NFC', u).encode('utf-8')

def obj_hash_dict(d):
    h = ''
    kh = [obj_hash(k) + obj_hash(v) for (k, v) in d.items()]
    for v in sorted(kh):
        h += v
    return hash_primitive('d', h)

def obj_hash_unicode(u):
    return hash_primitive('u', unicode_normalize(u))

def float_normalize(f):
    # special case 0
    # Note that if we allowed f to end up > .5 or == 0, we'd get the same thing
    if f == 0.0:
        return '+0:'
    # sign
    s = '+'
    if f < 0:
        s = '-'
        f = -f
    # exponent
    e = 0
    while f > 1:
        f /= 2
        e += 1
    while f <= .5:
        f *= 2
        e -= 1
    s += str(e) + ':'
    # mantissa
    assert f <= 1
    assert f > .5
    while f:
        if f >= 1:
            s += '1'
            f -= 1
        else:
            s += '0'
        assert f < 1
        assert len(s) < 1000
        f *= 2

    return s

def obj_hash_float(f):
    return hash_primitive('f', float_normalize(f))

def obj_hash_int(i):
    return hash_primitive('i', str(i))

def obj_hash_set(s):
    h = []
    for e in s:
        h.append(obj_hash(e))
    r = ''
    for t in sorted(h):
        r += t
    return hash_primitive('s', r)

class Redacted(object):
    def __init__(self, hash):
        self.hash = unhexify(hash)

class RedactedObject(Redacted):
    def __init__(self, o):
        self.hash = obj_hash(o)

def obj_hash(o):
    if type(o) is list or type(o) is FrozenList:
        return obj_hash_list(o)
    elif type(o) is dict:
        return obj_hash_dict(o)
    elif type(o) is unicode:
        return obj_hash_unicode(o)
    elif type(o) is float:
        return obj_hash_float(o)
    elif type(o) is int:
        return obj_hash_int(o)
    elif type(o) is str:
        return obj_hash_unicode(unicode(o))
    elif type(o) is set or type(o) is frozenset:
        return obj_hash_set(o)
    elif type(o) is bool:
        return obj_hash_bool(o)
    elif isinstance(o, Redacted):
        return o.hash
    elif o is None:
        return hash_primitive('n', '')

    print type(o)
    assert False

def python_json_hash(j):
    t = json.loads(j)
    return obj_hash(t)

def is_primitive_type(t):
    return t is str or t is unicode or t is float or t is int or t is bool or t is types.NoneType

class ApplyToLeaves(object):
    def __init__(self, leaf_fn, restrict = None):
        self.leaf_fn = leaf_fn
        self.restrict = restrict

    def __call__(self, o):
        t = type(o)
        if t is dict:
            return {self(k): self(v) for (k,v) in o.items()}
        elif t is list:
            return [self(e) for e in o]
        elif t is set:
            return set([self(e) for e in o])
        elif is_primitive_type(t):
            if self.restrict:
                if t in self.restrict:
                    return self.leaf_fn(o)
                return o
            else:
                return self.leaf_fn(o)

        print type(o)
        assert False

commonize = ApplyToLeaves(lambda o: float(o), (int,))

def common_json_hash(j):
    t = json.loads(j)
    t = commonize(t)
    return obj_hash(t)

def redactize_unicode(u):
    if u.startswith('**REDACTED**'):
        return Redacted(u[12:])
    else:
        return u

redactize = ApplyToLeaves(redactize_unicode, (str, unicode))

def common_redacted_json_hash(j):
    t = json.loads(j)
    t = commonize(t)
    t = redactize(t)
    return obj_hash(t)

class ApplyToLeavesAndKeys(ApplyToLeaves):
    def __init__(self, leaf_fn, key_fn):
        ApplyToLeaves.__init__(self, leaf_fn)
        self.key_fn = key_fn

    def __call__(self, o):
        if type(o) is dict:
            return {self.key_fn(k): self(v) for (k, v) in o.items()}
        return ApplyToLeaves.__call__(self, o)

def redactable_entity(e):
    return FrozenList([redactable_rand(), e])

def redactable_key(k):
    return redactable_rand() + k

def redactable_rand():
    r = ''
    for x in range(32):
        r += chr(random.SystemRandom().getrandbits(8))
    return hexify(r)

redactable = ApplyToLeavesAndKeys(redactable_entity, redactable_key)

def unredactable_dict(d):
    return {unredactable_key(k): unredactable(v) for (k, v) in d.items()}

def unredactable_key(k):
    return k[32:]

def unredactable_list(l):
    return [unredactable(e) for e in l]

def unredactable_set(s):
    return set([unredactable(e) for e in s])

def unredactable(o):
    t = type(o)
    if (t is list or t is FrozenList) and len(o) == 2 and type(o[0]) is str:
        assert is_primitive_type(type(o[1]))
        return o[1]
    elif t is dict:
        return unredactable_dict(o)
    elif t is list:
        return unredactable_list(o)
    elif t is set:
        return unredactable_set(o)

    print type(o)
    assert False

def unicode_normalize_entity(e):
    if type(e) is unicode:
        return unicode_normalize(e)
    elif type(e) is str:
        return unicode_normalize(unicode(e))
    return e

unicode_normalizify = ApplyToLeaves(unicode_normalize_entity)
<|endoftext|>


<commit_before>import pyramid
from sqlalchemy import (Column, Integer, Unicode, UnicodeText,
                        DateTime, ForeignKey)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm import relationship
from uuid import uuid1
from zope.sqlalchemy import ZopeTransactionExtension

DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
Base = declarative_base()

SAFE_MAX_LENGTH = 1000

def default_column():
    return Column(Unicode(SAFE_MAX_LENGTH))

class ModelMixin(object):

    def to_dict(self):
        result = {}
        for column in self.__class__.__mapper__.columns:
            if column.name in ('producer_id', 'location_id', 'event_id'):
                continue
            if column.name == 'id' and self.__class__.__name__ != 'Event':
                continue
            result[column.name] = {'value': getattr(self, column.name)}
        for collection_name in ('locations', 'dates'):
            if hasattr(self, collection_name):
                result[collection_name] = {'value': []}
                collection = getattr(self, collection_name)
                for obj in collection:
                    result[collection_name]['value'].append(obj.to_dict())
        return result

class Event(ModelMixin, Base):
    __tablename__ = 'events'
    id = Column(Integer, primary_key=True)

    audio_license = default_column()
    audio_url = default_column()
    author_email = default_column()
    author_firstname = default_column()
    author_lastname = default_column()
    author_telephone = default_column()
    description = Column(UnicodeText(SAFE_MAX_LENGTH))
    email = default_column()
    event_id = default_column()
    firstname = default_column()
    language = default_column()
    lastname = default_column()
    latlong = default_column()
    organiser = default_column()
    performers = default_column()
    photos1_license = default_column()
    photos1_url = default_column()
    photos2_license = default_column()
    photos2_url = default_column()
    press_url = default_column()
    price_information = default_column()
    source = default_column()
    source_id = default_column()
    target = default_column()
    telephone = default_column()
    title = default_column()
    uid = Column(Unicode(SAFE_MAX_LENGTH), unique=True)
    url = default_column()
    video_license = default_column()
    video_url = default_column()
    producer_id = default_column()

    locations = relationship('Location')

    def __init__(self, *args, **kwargs):
        if 'uid' not in kwargs:
            kwargs['uid'] = self.make_uid()
        locations = kwargs.get('locations', [])
        if locations:
            del kwargs['locations']
            for location_data in locations:
                dates = location_data.get('dates', [])
                location_kwargs = dict(location_data)
                del location_kwargs['dates']
                location = Location(**location_kwargs)
                self.locations.append(location)
                if dates:
                    del location_data['dates']
                    for date_data in dates:
                        date = Date(**date_data)
                        location.dates.append(date)
        super(Event, self).__init__(*args, **kwargs)

    def make_uid(self):
        return "{}@{}".format(
            uuid1().hex,
            pyramid.threadlocal.get_current_registry().settings['domain'],
        )

class Location(ModelMixin, Base):
    __tablename__ = 'locations'
    id = Column(Integer, primary_key=True)

    name = default_column()
    address = default_column()
    post_code = default_column()
    capacity = default_column()
    town = default_column()
    country = default_column()
    event_id = Column(Integer, ForeignKey('events.id'))
    dates = relationship('Date')

class Date(ModelMixin, Base):
    __tablename__ = 'dates'
    id = Column(Integer, primary_key=True)

    start_time = Column(DateTime(timezone=False))
    end_time = Column(DateTime(timezone=False))
    location_id = Column(Integer, ForeignKey('locations.id'))

class Source(ModelMixin, Base):
    __tablename__ = 'sources'
    id = Column(Integer, primary_key=True)
    url = default_column()
    producer_id = default_column()

icalendar_to_model_keys = {
    'uid': 'uid',
    'summary': 'title',
    'url': 'url',
    'description': 'description',
}

def flatten_values(mapping):
    result = {}
    for key, field in mapping.items():
        if field is None:
            continue
        else:
            result[key] = field['value']
    return result
<commit_msg>Split related collection initialization<commit_after>import pyramid
from sqlalchemy import (Column, Integer, Unicode, UnicodeText,
                        DateTime, ForeignKey)
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.orm import relationship
from uuid import uuid1
from zope.sqlalchemy import ZopeTransactionExtension

DBSession = scoped_session(sessionmaker(extension=ZopeTransactionExtension()))
Base = declarative_base()

SAFE_MAX_LENGTH = 1000

def default_column():
    return Column(Unicode(SAFE_MAX_LENGTH))

class ModelMixin(object):

    def to_dict(self):
        result = {}
        for column in self.__class__.__mapper__.columns:
            if column.name in ('producer_id', 'location_id', 'event_id'):
                continue
            if column.name == 'id' and self.__class__.__name__ != 'Event':
                continue
            result[column.name] = {'value': getattr(self, column.name)}
        for collection_name in ('locations', 'dates'):
            if hasattr(self, collection_name):
                result[collection_name] = {'value': []}
                collection = getattr(self, collection_name)
                for obj in collection:
                    result[collection_name]['value'].append(obj.to_dict())
        return result

class Event(ModelMixin, Base):
    __tablename__ = 'events'
    id = Column(Integer, primary_key=True)

    audio_license = default_column()
    audio_url = default_column()
    author_email = default_column()
    author_firstname = default_column()
    author_lastname = default_column()
    author_telephone = default_column()
    description = Column(UnicodeText(SAFE_MAX_LENGTH))
    email = default_column()
    event_id = default_column()
    firstname = default_column()
    language = default_column()
    lastname = default_column()
    latlong = default_column()
    organiser = default_column()
    performers = default_column()
    photos1_license = default_column()
    photos1_url = default_column()
    photos2_license = default_column()
    photos2_url = default_column()
    press_url = default_column()
    price_information = default_column()
    source = default_column()
    source_id = default_column()
    target = default_column()
    telephone = default_column()
    title = default_column()
    uid = Column(Unicode(SAFE_MAX_LENGTH), unique=True)
    url = default_column()
    video_license = default_column()
    video_url = default_column()
    producer_id = default_column()

    locations = relationship('Location')

    def __init__(self, *args, **kwargs):
        if 'uid' not in kwargs:
            kwargs['uid'] = self.make_uid()
        locations = kwargs.get('locations', [])
        if locations:
            del kwargs['locations']
            self.append_locations(locations)
        super(Event, self).__init__(*args, **kwargs)

    def make_uid(self):
        return "{}@{}".format(
            uuid1().hex,
            pyramid.threadlocal.get_current_registry().settings['domain'],
        )

    def append_locations(self, appstruct):
        for location_struct in appstruct:
            dates = location_struct.get('dates', [])
            del location_struct['dates']
            location = Location(**location_struct)
            if dates:
                location.append_dates(dates)
            self.locations.append(location)

class Location(ModelMixin, Base):
    __tablename__ = 'locations'
    id = Column(Integer, primary_key=True)

    name = default_column()
    address = default_column()
    post_code = default_column()
    capacity = default_column()
    town = default_column()
    country = default_column()
    event_id = Column(Integer, ForeignKey('events.id'))
    dates = relationship('Date')

    def append_dates(self, appstruct):
        for date_struct in appstruct:
            date = Date(**date_struct)
            self.dates.append(date)

class Date(ModelMixin, Base):
    __tablename__ = 'dates'
    id = Column(Integer, primary_key=True)

    start_time = Column(DateTime(timezone=False))
    end_time = Column(DateTime(timezone=False))
    location_id = Column(Integer, ForeignKey('locations.id'))

class Source(ModelMixin, Base):
    __tablename__ = 'sources'
    id = Column(Integer, primary_key=True)
    url = default_column()
    producer_id = default_column()

icalendar_to_model_keys = {
    'uid': 'uid',
    'summary': 'title',
    'url': 'url',
    'description': 'description',
}

def flatten_values(mapping):
    result = {}
    for key, field in mapping.items():
        if field is None:
            continue
        else:
            result[key] = field['value']
    return result
<|endoftext|>


<commit_before>import glob
import json
import logging
import requests
from requests.exceptions import RequestException

import django
from django.core.management import call_command

logger = logging.getLogger(__name__)

def populate_higlass_data_directory(data_dir):
    """
    Download remote files specified by urls in the input.json file
    :param data_dir: <String> Path to directory to populate with data
    """
    with open("/data/input.json") as f:
        config_data = json.loads(f.read())

    for url in config_data["file_relationships"]:
        try:
            response = requests.get(url)
        except RequestException as e:
            logger.error(
                "Something went wrong while fetching file from %s : %s",
                url,
                e
            )
        else:
            with open('{}{}'.format(data_dir, url.split("/")[-1]), 'wb') as f:
                f.write(response.content)

def ingest_tilesets(data_dir):
    """
   Ingest previously downloaded files into higlass-server w/ django
   management command
   :param data_dir: <String> Path to directory populated with data to ingest
   """
    files_to_ingest = glob.glob(
        '{}*multires.*'.format(data_dir)
    )

    for filename in files_to_ingest:
        call_command(
            "ingest_tileset",
            filename="{}".format(filename),
            filetype=get_filetype(filename),
            datatype=get_datatype(filename)
        )

def get_datatype(filename):
    datatype_mapping = {
        "cool": "matrix",
        "hitile": "vector"
    }
    try:
        datatype = datatype_mapping[filename.split(".")[-1]]
    except KeyError:
        logger.error(
            "Could not determine datatype from filename: %s", filename
        )
    else:
        return datatype

def get_filetype(filename):
    filetype_mapping = {
        "cool": "cooler",
        "hitile": "hitile"
    }
    try:
        filetype = filetype_mapping[filename.split(".")[-1]]
    except KeyError:
        logger.error(
            "Could not determine filetype from filename: %s", filename
        )
    else:
        return filetype

if __name__ == '__main__':
    data_dir = "/tmp/"

    # Allows for django commands to run in a standalone script
    django.setup()

    populate_higlass_data_directory(data_dir)
    ingest_tilesets(data_dir)
<commit_msg>Update `on_startup/py` to utilize the `/refinery-data` directory<commit_after>import glob
import json
import logging
import requests
from requests.exceptions import RequestException

import django
from django.core.management import call_command

logger = logging.getLogger(__name__)

def populate_higlass_data_directory(data_dir):
    """
    Download remote files specified by urls in the input.json file
    :param data_dir: <String> Path to directory to populate with data
    """
    with open("/data/input.json") as f:
        config_data = json.loads(f.read())

    for url in config_data["file_relationships"]:
        try:
            response = requests.get(url)
        except RequestException as e:
            raise RuntimeError(
                "Something went wrong while fetching file from {} : {}".format(
                    url,
                    e
                )
            )
        else:
            with open('{}{}'.format(data_dir, url.split("/")[-1]), 'wb') as f:
                f.write(response.content)

def ingest_tilesets(data_dir):
    """
   Ingest previously downloaded files into higlass-server w/ django
   management command
   :param data_dir: <String> Path to directory populated with data to ingest
   """
    files_to_ingest = glob.glob(
        '{}*multires.*'.format(data_dir)
    )

    for filename in files_to_ingest:
        call_command(
            "ingest_tileset",
            filename="{}".format(filename),
            filetype=get_filetype(filename),
            datatype=get_datatype(filename)
        )

def get_datatype(filename):
    datatype_mapping = {
        "cool": "matrix",
        "hitile": "vector"
    }
    try:
        datatype = datatype_mapping[filename.split(".")[-1]]
    except KeyError as e:
        raise RuntimeError(
            "Could not determine datatype from filename: {}".format(
                filename
            )
        )
    else:
        return datatype

def get_filetype(filename):
    filetype_mapping = {
        "cool": "cooler",
        "hitile": "hitile"
    }
    try:
        filetype = filetype_mapping[filename.split(".")[-1]]
    except KeyError:
        raise RuntimeError(
            "Could not determine filetype from filename: {}".format(
                filename
            )
        )
    else:
        return filetype

if __name__ == '__main__':
    data_dir = "/refinery-data/"

    # Allows for django commands to run in a standalone script
    django.setup()

    populate_higlass_data_directory(data_dir)
    ingest_tilesets(data_dir)
<|endoftext|>


<commit_before>from __future__ import print_function

from util import *
import nbtencoder as nbt

import argparse
import sys, os

cprintconf.name = "One Command"
cprintconf.color = bcolors.PEACH

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mode", help="Uses manual mode", dest="mode", default="", choices=["m", "i"])
parser.add_argument("-f", "--command_file", help="File to load commands from", dest="filepath",
	default=None)
parser.add_argument("-C", "--no-copy", help="Don't copy the output command", dest="nocopy", action="store_true")
parser.add_argument("-q", "--quiet", help="Silence output", dest="quiet", action="store_true")
parser.add_argument("-v", "--verbose", help="Detailed output", dest="loud", action="store_true")
args = parser.parse_args()

if args.quiet:
	def cprint(*args, **kwargs):
		pass

if args.nocopy:
	class pyperclip:
		@staticmethod
		def copy(*args, **kwargs): pass
		@staticmethod
		def paste(*args, **kwargs): pass
else:
	import pyperclip

class Command:
	def __init__(self, cmd, init=False, conditional=False):
		self.cmd = cmd
		self.cond = conditional
		self.init = init
	def __str__(self):
		return self.cmd
	def prettystr(self):
		return format("{cmd}{init}{cond}",
			cmd = self.cmd,
			init = "\n  - Initialization" if self.init else "",
			cond = "\n  - Conditional" if self.cond else "",)

def generate_sand(command_obj, direction, block="chain_command_block", auto=True):
	return {
		"Block": block,
		"Data": direction+8 if command_obj.cond else direction,
		"Time": 1,
		"TileEntityData": {
			"Command": str(command_obj),
			"auto": int(auto)
		},
		"id": "FallingSand"
	}

def normal_sand(block, data=0):
	return {
		"Block": block,
		"Data": data,
		"Time": 1,
		"id": "FallingSand"
	}

def ride(entities):
	topmost = None
	absoluteTopmost = None

	for entity in entities:
		if topmost == None:
			absoluteTopmost = entity
		else:
			topmost["Riding"] = entity
		topmost = entity
	return absoluteTopmost

if __name__ == "__main__":
	cprint("""{peach}----------------------------------------{endc}
	  {cyan}TheDestruc7i0n{endc} and {golden}Wire Segal{endc}'s 1.9 One Command Creator
	 {green}Prepend your command with `INIT:` to make it only run when the structure is deployed.{endc}
	 {green}Prepend your command with `COND:` to make it a conditional command.{endc}
	        Please report any bugs at the GitHub repo: {line}{blue}https://github.com/destruc7i0n/OneCommand/issues{endc}
	        {peach}----------------------------------------{endc}""", strip=True)

	# get mode if not specified by argument
	if not args.mode:
		mode = cinput("Manual (m) or Instant (i)? ").strip().rstrip().lower()
		if mode not in ["m", "i"]:
			raise ValueError("Not manual or instant")
	else:
		mode = args.mode

	commands = []
	# get commands if file not specified
	if not args.filepath:
		x = 1
		command = cinput("Command {num}: ", num=x).strip().rstrip()
		while command:
			x += 1
			commands.append(command)
			command = cinput("Command {num}: ", num=x).strip().rstrip()
	# get commands from specified file
	else:
		if os.path.exists(args.filepath):
			commands = open(args.filepath).read().split("\n")
		else:
			raise IOError(format("File {file} not found.", file=args.filepath))

	init_commands = []
	clock_commands = []
	# do all INIT and COND checking
	for command in commands:
		if not command: continue
		command = command.strip().rstrip()
		if command[0] == "#": continue
		init = False
		conditional = False
		while command[:5] in ["INIT:","COND:"]:
			if command[:5] == "COND:": conditional = True
			elif command[:5] == "INIT:": init = True
			command = command[5:]
		command_obj = Command(command, conditional=conditional, init=init)
		if init:
			init_commands.append(command_obj)
		else:
			clock_commands.append(command_obj)

	final_command_obj = None
	if clock_commands or init_commands:
		command_sands = []

		for command in init_commands:
			if args.loud:
				cprint(command.prettystr())
			if command is init_commands[0]:
				command_sands.append(generate_sand(command, 0, "command_block"))
			else:
				command_sands.append(generate_sand(command, 0))

		if mode == 'i' and clock_commands:
			block = "command_block" if not init_commands else "chain_command_block"
			offset = len(clock_commands) + 1
			blockdata = Command(format("blockdata ~ ~-{offset} ~ {auto:1b}", offset = offset), init=True)
			if args.loud:
				cprint(blockdata.prettystr())
			command_sands.append(generate_sand(blockdata, 0, block))

		offset = len(init_commands) + int(bool(mode == 'i' and clock_commands))
		if offset:
			fill = Command(format("fill ~ ~-1 ~ ~ ~{offset} ~ air", offset = offset), init=True)
			if args.loud:
				cprint(fill.prettystr())
			command_sands.append(generate_sand(fill, 0))
			command_sands.append(normal_sand("barrier"))

		for command in clock_commands[::-1]:
			if args.loud:
				cprint(command.prettystr())
			if command is clock_commands[0]:
				command_sands.append(generate_sand(command, 1, "repeating_command_block", mode == 'i'))
			else:
				command_sands.append(generate_sand(command, 1))
		final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", ride(command_sands))

	final_command = nbt.JSON2Command(final_command_obj)

	if len(final_command) <= 32500 and final_command:
		pyperclip.copy(final_command)
		if args.nocopy:
			cprint("{bold}Final command - {endc}", func=sys.stderr.write)
		else:
			cprint("{bold}Copied to clipboard - {endc}", func=sys.stderr.write)
		sys.stdout.write(final_command + "\n")
	elif not final_command:
		cprint("No command generated.", color=bcolors.RED)
	else:
		cprint("Command too large ({length} > 32500)", length=len(final), color=bcolors.RED)
<commit_msg>change in log<commit_after>from __future__ import print_function

from util import *
import nbtencoder as nbt

import argparse
import sys, os

cprintconf.name = "Generator"
cprintconf.color = bcolors.PEACH

parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mode", help="Uses manual mode", dest="mode", default="", choices=["m", "i"])
parser.add_argument("-f", "--command_file", help="File to load commands from", dest="filepath",
	default=None)
parser.add_argument("-C", "--no-copy", help="Don't copy the output command", dest="nocopy", action="store_true")
parser.add_argument("-q", "--quiet", help="Silence output", dest="quiet", action="store_true")
parser.add_argument("-v", "--verbose", help="Detailed output", dest="loud", action="store_true")
args = parser.parse_args()

if args.quiet:
	def cprint(*args, **kwargs):
		pass

if args.nocopy:
	class pyperclip:
		@staticmethod
		def copy(*args, **kwargs): pass
		@staticmethod
		def paste(*args, **kwargs): pass
else:
	import pyperclip

class Command:
	def __init__(self, cmd, init=False, conditional=False):
		self.cmd = cmd
		self.cond = conditional
		self.init = init
	def __str__(self):
		return self.cmd
	def prettystr(self):
		return format("{cmd}{init}{cond}",
			cmd = self.cmd,
			init = "\n  - Initialization" if self.init else "",
			cond = "\n  - Conditional" if self.cond else "",)

def generate_sand(command_obj, direction, block="chain_command_block", auto=True):
	return {
		"Block": block,
		"Data": direction+8 if command_obj.cond else direction,
		"Time": 1,
		"TileEntityData": {
			"Command": str(command_obj),
			"auto": int(auto)
		},
		"id": "FallingSand"
	}

def normal_sand(block, data=0):
	return {
		"Block": block,
		"Data": data,
		"Time": 1,
		"id": "FallingSand"
	}

def ride(entities):
	topmost = None
	absoluteTopmost = None

	for entity in entities:
		if topmost == None:
			absoluteTopmost = entity
		else:
			topmost["Riding"] = entity
		topmost = entity
	return absoluteTopmost

if __name__ == "__main__":
	cprint("""{peach}----------------------------------------{endc}
	  {cyan}TheDestruc7i0n{endc} and {golden}Wire Segal{endc}'s 1.9 One Command Generator
	 {green}Prepend your command with `INIT:` to make it only run when the structure is deployed.{endc}
	 {green}Prepend your command with `COND:` to make it a conditional command.{endc}
	        Please report any bugs at the GitHub repo: {line}{blue}https://github.com/destruc7i0n/OneCommand/issues{endc}
	        {peach}----------------------------------------{endc}""", strip=True)

	# get mode if not specified by argument
	if not args.mode:
		mode = cinput("Manual (m) or Instant (i)? ").strip().rstrip().lower()
		if mode not in ["m", "i"]:
			raise ValueError("Not manual or instant")
	else:
		mode = args.mode

	commands = []
	# get commands if file not specified
	if not args.filepath:
		x = 1
		command = cinput("Command {num}: ", num=x).strip().rstrip()
		while command:
			x += 1
			commands.append(command)
			command = cinput("Command {num}: ", num=x).strip().rstrip()
	# get commands from specified file
	else:
		if os.path.exists(args.filepath):
			commands = open(args.filepath).read().split("\n")
		else:
			raise IOError(format("File {file} not found.", file=args.filepath))

	init_commands = []
	clock_commands = []
	# do all INIT and COND checking
	for command in commands:
		if not command: continue
		command = command.strip().rstrip()
		if command[0] == "#": continue
		init = False
		conditional = False
		while command[:5] in ["INIT:","COND:"]:
			if command[:5] == "COND:": conditional = True
			elif command[:5] == "INIT:": init = True
			command = command[5:]
		command_obj = Command(command, conditional=conditional, init=init)
		if init:
			init_commands.append(command_obj)
		else:
			clock_commands.append(command_obj)

	final_command_obj = None
	if clock_commands or init_commands:
		command_sands = []

		for command in init_commands:
			if args.loud:
				cprint(command.prettystr())
			if command is init_commands[0]:
				command_sands.append(generate_sand(command, 0, "command_block"))
			else:
				command_sands.append(generate_sand(command, 0))

		if mode == 'i' and clock_commands:
			block = "command_block" if not init_commands else "chain_command_block"
			offset = len(clock_commands) + 1
			blockdata = Command(format("blockdata ~ ~-{offset} ~ {auto:1b}", offset = offset), init=True)
			if args.loud:
				cprint(blockdata.prettystr())
			command_sands.append(generate_sand(blockdata, 0, block))

		offset = len(init_commands) + int(bool(mode == 'i' and clock_commands))
		if offset:
			fill = Command(format("fill ~ ~-1 ~ ~ ~{offset} ~ air", offset = offset), init=True)
			if args.loud:
				cprint(fill.prettystr())
			command_sands.append(generate_sand(fill, 0))
			command_sands.append(normal_sand("barrier"))

		for command in clock_commands[::-1]:
			if args.loud:
				cprint(command.prettystr())
			if command is clock_commands[0]:
				command_sands.append(generate_sand(command, 1, "repeating_command_block", mode == 'i'))
			else:
				command_sands.append(generate_sand(command, 1))
		final_command_obj = nbt.cmd("summon FallingSand ~ ~1 ~ ", ride(command_sands))

	final_command = nbt.JSON2Command(final_command_obj)

	if len(final_command) <= 32500 and final_command:
		pyperclip.copy(final_command)
		if args.nocopy:
			cprint("{bold}Final command - {endc}", func=sys.stderr.write)
		else:
			cprint("{bold}Copied to clipboard - {endc}", func=sys.stderr.write)
		sys.stdout.write(final_command + "\n")
	elif not final_command:
		cprint("No command generated.", color=bcolors.RED)
	else:
		cprint("Command too large ({length} > 32500)", length=len(final), color=bcolors.RED)
<|endoftext|>


<commit_before>import logging
log = logging.getLogger()
logging.basicConfig(format='[%(levelname)-8s] %(message)s')
log.setLevel(logging.DEBUG)

import urllib2
from urllib import urlencode
try:
    import ujson as json
except ImportError:
    log.warn("Could not import ujson; falling back to regular (slow) json")
    import json

ENDPOINT = 'http://api.onebusaway.org/api/'

class OneBusAway:
    def __init__(self, key):
        self._key = key

    def stopsForLocation(self, lat, lon, radius=250):
        args = locals()
        del args['self']
        args['key'] = self._key
        URI = '%swhere/stops-for-location.json?%s' % (ENDPOINT, urlencode(args))
        data = json.load(urllib2.urlopen(URI))
        stops = data['data']['stops']
        return [str(y[1]) for y in sorted([((x['lat']-lat)**2+(x['lon']-lon)**2, x['id']+":"+x['direction']+":"+x['name']) for x in stops], key=lambda x:x[0])]

    def arrivalsAndDeparturesForStop(self, stop):
        URI = '%swhere/arrivals-and-departures-for-stop/%s.json?key=%s' % (ENDPOINT, stop, self._key)
        data = json.load(urllib2.urlopen(URI))
        ctime = data['currentTime']
        arrivals = data['data']['arrivalsAndDepartures']
        for x in arrivals:
            at = x['predictedArrivalTime']
            if at == 0:
                at = x['scheduledArrivalTime']
            x['minutesToArrival'] = ((at - ctime)/6000+5)/10
        return sorted([(x['routeShortName'], x['minutesToArrival']) for x in arrivals], key=lambda x:x[1])
<commit_msg>Changing ampersand to newline.<commit_after>import logging
log = logging.getLogger()
logging.basicConfig(format='[%(levelname)-8s] %(message)s')
log.setLevel(logging.DEBUG)

import urllib2
from urllib import urlencode
try:
    import ujson as json
except ImportError:
    log.warn("Could not import ujson; falling back to regular (slow) json")
    import json

ENDPOINT = 'http://api.onebusaway.org/api/'

class OneBusAway:
    def __init__(self, key):
        self._key = key

    def stopsForLocation(self, lat, lon, radius=250):
        args = locals()
        del args['self']
        args['key'] = self._key
        URI = '%swhere/stops-for-location.json?%s' % (ENDPOINT, urlencode(args))
        data = json.load(urllib2.urlopen(URI))
        stops = data['data']['stops']
        return [str(y[1]) for y in sorted([((x['lat']-lat)**2+(x['lon']-lon)**2, x['id']+":"+x['direction']+":"+x['name'].replace(" & ", "\n") for x in stops], key=lambda x:x[0])]

    def arrivalsAndDeparturesForStop(self, stop):
        URI = '%swhere/arrivals-and-departures-for-stop/%s.json?key=%s' % (ENDPOINT, stop, self._key)
        data = json.load(urllib2.urlopen(URI))
        ctime = data['currentTime']
        arrivals = data['data']['arrivalsAndDepartures']
        for x in arrivals:
            at = x['predictedArrivalTime']
            if at == 0:
                at = x['scheduledArrivalTime']
            x['minutesToArrival'] = ((at - ctime)/6000+5)/10
        return sorted([(x['routeShortName'], x['minutesToArrival']) for x in arrivals], key=lambda x:x[1])
<|endoftext|>


<commit_before>import os, sys, getopt, configparser, datetime
import htmlReportGen
from collections import OrderedDict

possible_params = []            #Possible parameters they could input
parameters = {}                 #Dictionary that holds the parameters:their filters

outputDir, inputDir = '', ''    #Folder where to report, folder where logs are

front_extra = ' data_type="4">' #Needed for parsing xml values, the front tags end
back_extra = '</'               #Needed for parsing xml values, the back tags front

values, count = [], []          #Values holds the lines in the report
                                #Count is how many times an identical line appeared

start_time, end_time = "", ""   #Start and end time frames given by user

#When -h is called or the user doesn't do something right
helpfile="""
               ,   .,---.,---.          |
,---.,---.,---.|\  ||---'`---.,---.,---.|__/ ,---.
|   ||   ||---'| \ ||        ||   |,---||  \ |---'
`---'|---'`---'`  `'`    `---'`   '`---^`   ``---'
     |

Parses NPS logs and generates useful reports

Usage: python opeNPSnake.py -i "filepath" [options]

Options:
    -h Prints out this help file
    -i Input file/directory (YOU MUST QUOTE THE FILE PATH)
    -o Output directory (Defaults to the current working directory)
    -P Prints list of log parameterss
    -p Select parameters for parsing [-p arg1:filter1,arg2:filter2:filter3,arg3]
    -c Specifies config file (see sample.conf)
    -t Specify the time frame [-t "* * * * *,* * * * *"] Year, Month, Day, Hour, Minute

Note: Fully-Qualifed-User-Names is not spelled correctly in the logs.

"""

#Gets the xml values from the logs
#Expects the line of text from the log, the starting <' '> tag and ending </' '> tag
def get_xml_value(line, start_tag, end_tag):
    x = line.find(start_tag)
    if x == -1:                     #Returns N/A if that tag isn't there
        return "N/A"
    x += len(start_tag + front_extra)
    y = line.find(end_tag)
    return line[x:y]

#Parses the files and pulls the desired values
def parseFiles():
    for file in os.listdir(inputDir):
        if file.endswith('.log'):
            print("Parsing data from file: " + file)
            inputFile = open(inputDir + file)
            lines = inputFile.readlines()       #List of lines in the file
            for line in lines:
                #Temp variable to store the values that will be reported
                values_temp = []
                #Boolean for checking if a filter has been broken
                #False = the values get reported
                #True = A filter has been broken
                broken_filter = False
                for param in parameters:
                    #The front <' '> tag
                    start_tag = param
                    #The back </' '> tag
                    end_tag = back_extra + param
                    #temp storage for the values
                    xml_value = get_xml_value(line, start_tag, end_tag)
                    #Checks to see if it gets passed the filter or there is no filter
                    if xml_value in parameters[param] or len(parameters[param]) <= 0:
                        values_temp.append(xml_value)
                    else:
                        broken_filter = True
                if not broken_filter:
                    #Checks to see if this is a duplicate line
                    if values_temp not in values:
                        values.append(values_temp)
                        count.append(1)
                    #Adds one to count if it is
                    else:
                        index = values.index(values_temp)
                        count[index] += 1
            print(values)
            inputFile.close()

#Parses the files for possible parameter types
def checkFilesForParameters():
    for file in os.listdir(inputDir):
        if file.endswith('.log'):
            inputfile = open(inputDir + file)
            #List that holds all the lines in the file
            lines = inputfile.readlines()
            #temp variable to hold a place in a line so we progress through the line
            lastindex = 0
            for line in lines:
                while lastindex != -1:
                    #Where the last item was
                    lastindex = line.find(back_extra, lastindex)
                    #Next items first <' '>
                    fbracket = int(line.find(back_extra, lastindex)) + 2
                    #Next items second </' '>
                    sbracket = int(line.find('>', lastindex))
                    #Break if there are no more values
                    if sbracket == -1:
                        break
                    param = line[fbracket:sbracket]
                    #No duplicates
                    if param not in possible_params:
                        possible_params.append(param)
                    lastindex+=1
            inputfile.close()
    #These variable break or are specified through another cmdline argument
    for param in ["Timestamp", "User-Name", "Event"]:
        possible_params.remove(param)

#Returns a reformatted folder path
def getFolderPath(path):
    #Replace the \ slashes with / so we don't get unicode errors
    temp = path.replace('\\', '/')
    #Add a / to the end if there isn't one
    if temp[-1:] != "/":
        temp += "/"
    return temp

#Checks the parameters that the user specified and drops the ones that
#Aren't in the logs
def getParameters(params):
    checkFilesForParameters()
    global parameters
    for p in params:
        if p not in parameters and p in possible_params:
            parameters[p] = params[p]
        else:
            print(p + " is not a valid parameter")

#Returns a dictionary {key:value} = {parameter:[filter1, filter2, etc]}
def getFilters(arg):
    params = arg.split(',')
    paramlst = {}
    for p in params:
        if p[0] == " ":
            p = p[1:]
        filterlst=[]
        try:
            filterlst=p.split(':')[1:]
        except:
            filterlst.append('')

        paramlst[p.split(':')[0].lower().replace(' ', '-').title()]=filterlst
    print(paramlst)
    return paramlst

#loads config file
def loadConf(loc):
    global inputDir,outputDir
    loc = loc.replace('\\', '/')
    c = configparser.ConfigParser()
    c.read(loc)
    section='Config'
    options=c.options(section)
    paramlst = {}
    for o in options:
        if o == 'parameters':
            arg = c.get(section,o)
            paramlst = getFilters(arg)
        if o == 'input':
            inputDir = getFolderPath(c.get(section,o))
        if o == 'output':
            outputDir = getFolderPath(c.get(section,o))
    getParameters(paramlst)

#convert user supplied date to correct format
def convertDate(times):
    time_start_split = times[0].split(' ')
    time_start_parsed=[]
    for s in time_start_split:
        if s == '*':
            s = 1
        else:
            s = int(s)
        time_start_parsed.append(s)
    temp_start = datetime.datetime(time_start_parsed[0], time_start_parsed[1], time_start_parsed[2], time_start_parsed[3], time_start_parsed[4])
    time_end_split = times[1].split(' ')
    time_end_parsed=[]
    for s in time_end_split:
        if s == '*':
            s = 1
        else:
            s = int(s)
        time_end_parsed.append(s)
    temp_end = datetime.datetime(time_end_parsed[0], time_end_parsed[1], time_end_parsed[2], time_end_parsed[3], time_end_parsed[4])
    return temp_start,temp_end

#Main
#We should probably split this up but whatever
def main():
    global filters
    global outputDir, inputDir
    global start_time, end_time
    #get the cmd line options
    #try:
    opts, args = getopt.getopt(sys.argv[1:],'hi:o:Pp:t:c:')
    #except:
    #    print(helpfile)
    #do stuff based on options
    for opt, arg in opts:
        paramlst = {}
        #print help file
        if opt == '-h':
            print(helpfile)
        #get input directory
        elif opt == '-i':
            inputDir = getFolderPath(arg)
        #get output directory
        elif opt == '-o':
            outputDir = getFolderPath(arg)
        #prints out list of parameters
        elif opt == '-P':
            checkFilesForParameters()
            for param in possible_params:
                print(param.replace("-", " "))
        #selects parameters for parsing
        elif opt == '-p':
            paramlst = getFilters(arg)
        #specifies the time frame
        elif opt == '-t':
            times = arg.split(',')
            parameters['Timestamp']=''
            global start_time,end_time
            start_time,end_time=convertDate(times)

        #Load parameters/filters from a config file
        elif opt == '-c':
            loadConf(arg)
    #Make sure they specified a -p parameter
    getParameters(paramlst)
    if len(parameters) > 0:
        parseFiles()
        ordParameters = OrderedDict(parameters)
        print(ordParameters)
        #stupid way to check if -t
        if 'Timestamp' in ordParameters:
            ordParameters.move_to_end("Timestamp")
            #take out everything except events in specified time range
            for v in values:
                print(v)
                date = v[parameters.]
                dt = datetime.datetime(int(date.split('/')[2].split(" ")[0]),int(date.split('/')[0]),int(date.split('/')[1]),int(date.split(' ')[1].split(':')[0]),int(date.split(':')[1]))
        #Generating the reports
        #If there wasn't a specified outputDir we just use the default(cwd)
        if outputDir == '':
            htmlReportGen.generate(values, ordParameters, count)
        else:
            htmlReportGen.generate(values, ordParameters, count, outputDir)

    elif ('-P', '') not in opts:
        print(helpfile)
        print("You did not specify any parameters")

if __name__ == '__main__':
    main()
<commit_msg>I DID IT<commit_after>import os, sys, getopt, configparser, datetime
import htmlReportGen
from collections import OrderedDict

possible_params = []            #Possible parameters they could input
parameters = {}                 #Dictionary that holds the parameters:their filters

outputDir, inputDir = '', ''    #Folder where to report, folder where logs are

front_extra = ' data_type="4">' #Needed for parsing xml values, the front tags end
back_extra = '</'               #Needed for parsing xml values, the back tags front

values, count = [], []          #Values holds the lines in the report
                                #Count is how many times an identical line appeared

start_time, end_time = "", ""   #Start and end time frames given by user

#When -h is called or the user doesn't do something right
helpfile="""
               ,   .,---.,---.          |
,---.,---.,---.|\  ||---'`---.,---.,---.|__/ ,---.
|   ||   ||---'| \ ||        ||   |,---||  \ |---'
`---'|---'`---'`  `'`    `---'`   '`---^`   ``---'
     |

Parses NPS logs and generates useful reports

Usage: python opeNPSnake.py -i "filepath" [options]

Options:
    -h Prints out this help file
    -i Input file/directory (YOU MUST QUOTE THE FILE PATH)
    -o Output directory (Defaults to the current working directory)
    -P Prints list of log parameterss
    -p Select parameters for parsing [-p arg1:filter1,arg2:filter2:filter3,arg3]
    -c Specifies config file (see sample.conf)
    -t Specify the time frame [-t "* * * * *,* * * * *"] Year, Month, Day, Hour, Minute

Note: Fully-Qualifed-User-Names is not spelled correctly in the logs.

"""

#Gets the xml values from the logs
#Expects the line of text from the log, the starting <' '> tag and ending </' '> tag
def get_xml_value(line, start_tag, end_tag):
    x = line.find(start_tag)
    if x == -1:                     #Returns N/A if that tag isn't there
        return "N/A"
    x += len(start_tag + front_extra)
    y = line.find(end_tag)
    return line[x:y]

#Parses the files and pulls the desired values
def parseFiles():
    for file in os.listdir(inputDir):
        if file.endswith('.log'):
            print("Parsing data from file: " + file)
            inputFile = open(inputDir + file)
            lines = inputFile.readlines()       #List of lines in the file
            for line in lines:
                #Temp variable to store the values that will be reported
                values_temp = []
                #Boolean for checking if a filter has been broken
                #False = the values get reported
                #True = A filter has been broken
                broken_filter = False
                for param in parameters:
                    #The front <' '> tag
                    start_tag = param
                    #The back </' '> tag
                    end_tag = back_extra + param
                    #temp storage for the values
                    xml_value = get_xml_value(line, start_tag, end_tag)
                    #Checks to see if it gets passed the filter or there is no filter
                    if xml_value in parameters[param] or len(parameters[param]) <= 0:
                        values_temp.append(xml_value)
                    else:
                        broken_filter = True
                if not broken_filter:
                    #Checks to see if this is a duplicate line
                    if values_temp not in values:
                        values.append(values_temp)
                        count.append(1)
                    #Adds one to count if it is
                    else:
                        index = values.index(values_temp)
                        count[index] += 1
            print(values)
            inputFile.close()

#Parses the files for possible parameter types
def checkFilesForParameters():
    for file in os.listdir(inputDir):
        if file.endswith('.log'):
            inputfile = open(inputDir + file)
            #List that holds all the lines in the file
            lines = inputfile.readlines()
            #temp variable to hold a place in a line so we progress through the line
            lastindex = 0
            for line in lines:
                while lastindex != -1:
                    #Where the last item was
                    lastindex = line.find(back_extra, lastindex)
                    #Next items first <' '>
                    fbracket = int(line.find(back_extra, lastindex)) + 2
                    #Next items second </' '>
                    sbracket = int(line.find('>', lastindex))
                    #Break if there are no more values
                    if sbracket == -1:
                        break
                    param = line[fbracket:sbracket]
                    #No duplicates
                    if param not in possible_params:
                        possible_params.append(param)
                    lastindex+=1
            inputfile.close()
    #These variable break or are specified through another cmdline argument
    for param in ["Timestamp", "User-Name", "Event"]:
        possible_params.remove(param)

#Returns a reformatted folder path
def getFolderPath(path):
    #Replace the \ slashes with / so we don't get unicode errors
    temp = path.replace('\\', '/')
    #Add a / to the end if there isn't one
    if temp[-1:] != "/":
        temp += "/"
    return temp

#Checks the parameters that the user specified and drops the ones that
#Aren't in the logs
def getParameters(params):
    checkFilesForParameters()
    global parameters
    for p in params:
        if p not in parameters and p in possible_params:
            parameters[p] = params[p]
        else:
            print(p + " is not a valid parameter")

#Returns a dictionary {key:value} = {parameter:[filter1, filter2, etc]}
def getFilters(arg):
    params = arg.split(',')
    paramlst = {}
    for p in params:
        if p[0] == " ":
            p = p[1:]
        filterlst=[]
        try:
            filterlst=p.split(':')[1:]
        except:
            filterlst.append('')

        paramlst[p.split(':')[0].lower().replace(' ', '-').title()]=filterlst
    print(paramlst)
    return paramlst

#loads config file
def loadConf(loc):
    global inputDir,outputDir
    loc = loc.replace('\\', '/')
    c = configparser.ConfigParser()
    c.read(loc)
    section='Config'
    options=c.options(section)
    paramlst = {}
    for o in options:
        if o == 'parameters':
            arg = c.get(section,o)
            paramlst = getFilters(arg)
        if o == 'input':
            inputDir = getFolderPath(c.get(section,o))
        if o == 'output':
            outputDir = getFolderPath(c.get(section,o))
    getParameters(paramlst)

#convert user supplied date to correct format
def convertDate(times):
    time_start_split = times[0].split(' ')
    time_start_parsed=[]
    for s in time_start_split:
        if s == '*':
            s = 1
        else:
            s = int(s)
        time_start_parsed.append(s)
    temp_start = datetime.datetime(time_start_parsed[0], time_start_parsed[1], time_start_parsed[2], time_start_parsed[3], time_start_parsed[4])
    time_end_split = times[1].split(' ')
    time_end_parsed=[]
    for s in time_end_split:
        if s == '*':
            s = 1
        else:
            s = int(s)
        time_end_parsed.append(s)
    temp_end = datetime.datetime(time_end_parsed[0], time_end_parsed[1], time_end_parsed[2], time_end_parsed[3], time_end_parsed[4])
    return temp_start,temp_end

#Main
#We should probably split this up but whatever
def main():
    global filters
    global outputDir, inputDir
    global start_time, end_time
    #get the cmd line options
    #try:
    opts, args = getopt.getopt(sys.argv[1:],'hi:o:Pp:t:c:')
    #except:
    #    print(helpfile)
    #do stuff based on options
    for opt, arg in opts:
        paramlst = {}
        #print help file
        if opt == '-h':
            print(helpfile)
        #get input directory
        elif opt == '-i':
            inputDir = getFolderPath(arg)
        #get output directory
        elif opt == '-o':
            outputDir = getFolderPath(arg)
        #prints out list of parameters
        elif opt == '-P':
            checkFilesForParameters()
            for param in possible_params:
                print(param.replace("-", " "))
        #selects parameters for parsing
        elif opt == '-p':
            paramlst = getFilters(arg)
        #specifies the time frame
        elif opt == '-t':
            times = arg.split(',')
            parameters['Timestamp']=''
            global start_time,end_time
            start_time,end_time=convertDate(times)

        #Load parameters/filters from a config file
        elif opt == '-c':
            loadConf(arg)
    #Make sure they specified a -p parameter
    getParameters(paramlst)
    if len(parameters) > 0:
        parseFiles()
        ordParameters = OrderedDict(parameters)
        print(ordParameters)
        #stupid way to check if -t
        if 'Timestamp' in ordParameters:
            ordParameters.move_to_end("Timestamp")
            #take out everything except events in specified time range
            for v in values:
                print(v)
                date = v[list(parameters.keys()).index("Timestamp")]
                dt = datetime.datetime(int(date.split('/')[2].split(" ")[0]),int(date.split('/')[0]),int(date.split('/')[1]),int(date.split(' ')[1].split(':')[0]),int(date.split(':')[1]))
        #Generating the reports
        #If there wasn't a specified outputDir we just use the default(cwd)
        if outputDir == '':
            htmlReportGen.generate(values, ordParameters, count)
        else:
            htmlReportGen.generate(values, ordParameters, count, outputDir)

    elif ('-P', '') not in opts:
        print(helpfile)
        print("You did not specify any parameters")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import os
import struct
from fractions import Fraction
import importlib

from litex.gen import *
from litex.gen.fhdl.specials import Keep
from litex.gen.genlib.io import CRG
from litex.gen.genlib.resetsync import AsyncResetSynchronizer
from litex.gen.genlib.misc import WaitTimer

from litex.soc.cores.sdram.settings import MT41J128M16
from litex.soc.cores.sdram.phy import s6ddrphy
from litex.soc.integration.soc_core import mem_decoder
from litex.soc.integration.soc_sdram import *
from litex.soc.integration.builder import *
from litex.soc.cores.gpio import GPIOIn, GPIOOut
from litex.soc.interconnect.csr import AutoCSR
from litex.soc.cores.uart.bridge import UARTWishboneBridge
from litex.soc.cores.sdram.lasmicon.core import ControllerSettings

from liteeth.phy.s6rgmii import LiteEthPHYRGMII
from liteeth.core.mac import LiteEthMAC

import opsis_platform

from gateware import dna
from gateware import firmware

def csr_map_update(csr_map, csr_peripherals):
  csr_map.update(dict((n, v) for v, n in enumerate(csr_peripherals, start=max(csr_map.values()) + 1)))

class FrontPanelGPIO(Module, AutoCSR):
    def __init__(self, platform, clk_freq):
        switches = Signal(1)
        leds = Signal(2)

        self.reset = Signal()

        # # #

        self.submodules.switches = GPIOIn(switches)
        self.submodules.leds = GPIOOut(leds)
        self.comb += [
           switches[0].eq(~platform.request("pwrsw")),
           platform.request("hdled").eq(~leds[0]),
           platform.request("pwled").eq(~leds[1]),
        ]

        # generate a reset when power switch is pressed for 1 second
        self.submodules.reset_timer = WaitTimer(clk_freq)
        self.comb += [
            self.reset_timer.wait.eq(switches[0]),
            self.reset.eq(self.reset_timer.done)
        ]

class _CRG(Module):
    def __init__(self, platform, clk_freq):
        self.clock_domains.cd_sys = ClockDomain()
        self.clock_domains.cd_sys2x = ClockDomain()
        self.clock_domains.cd_sdram_half = ClockDomain()
        self.clock_domains.cd_sdram_full_wr = ClockDomain()
        self.clock_domains.cd_sdram_full_rd = ClockDomain()
        self.clock_domains.cd_base50 = ClockDomain()

        self.clk8x_wr_strb = Signal()
        self.clk8x_rd_strb = Signal()

        self.reset = Signal()

        f0 = 100*1000000
        clk100 = platform.request("clk100")
        clk100a = Signal()
        self.specials += Instance("IBUFG", i_I=clk100, o_O=clk100a)
        clk100b = Signal()
        self.specials += Instance("BUFIO2", p_DIVIDE=1,
                                  p_DIVIDE_BYPASS="TRUE", p_I_INVERT="FALSE",
                                  i_I=clk100a, o_DIVCLK=clk100b)
        f = Fraction(int(clk_freq), int(f0))
        n, m = f.denominator, f.numerator
        assert f0/n*m == clk_freq
        p = 8
        pll_lckd = Signal()
        pll_fb = Signal()
        pll = Signal(6)
        self.specials.pll = Instance("PLL_ADV", p_SIM_DEVICE="SPARTAN6",
                                     p_BANDWIDTH="OPTIMIZED", p_COMPENSATION="INTERNAL",
                                     p_REF_JITTER=.01, p_CLK_FEEDBACK="CLKFBOUT",
                                     i_DADDR=0, i_DCLK=0, i_DEN=0, i_DI=0, i_DWE=0, i_RST=0, i_REL=0,
                                     p_DIVCLK_DIVIDE=1, p_CLKFBOUT_MULT=m*p//n, p_CLKFBOUT_PHASE=0.,
                                     i_CLKIN1=clk100b, i_CLKIN2=0, i_CLKINSEL=1,
                                     p_CLKIN1_PERIOD=1e9/f0, p_CLKIN2_PERIOD=0.,
                                     i_CLKFBIN=pll_fb, o_CLKFBOUT=pll_fb, o_LOCKED=pll_lckd,
                                     o_CLKOUT0=pll[0], p_CLKOUT0_DUTY_CYCLE=.5,
                                     o_CLKOUT1=pll[1], p_CLKOUT1_DUTY_CYCLE=.5,
                                     o_CLKOUT2=pll[2], p_CLKOUT2_DUTY_CYCLE=.5,
                                     o_CLKOUT3=pll[3], p_CLKOUT3_DUTY_CYCLE=.5,
                                     o_CLKOUT4=pll[4], p_CLKOUT4_DUTY_CYCLE=.5,
                                     o_CLKOUT5=pll[5], p_CLKOUT5_DUTY_CYCLE=.5,
                                     p_CLKOUT0_PHASE=0., p_CLKOUT0_DIVIDE=p//8,    # sdram wr/rd full clock
                                     p_CLKOUT1_PHASE=0., p_CLKOUT1_DIVIDE=p//8,    # unused
                                     p_CLKOUT2_PHASE=230., p_CLKOUT2_DIVIDE=p//4,  # sdram dqs adr ctrl off-chip
                                     p_CLKOUT3_PHASE=210., p_CLKOUT3_DIVIDE=p//4,  # ddr half clock
                                     p_CLKOUT4_PHASE=0., p_CLKOUT4_DIVIDE=p//2,    # 2x system clock
                                     p_CLKOUT5_PHASE=0., p_CLKOUT5_DIVIDE=p//1,    # system clock
        )
        self.specials += Instance("BUFG", i_I=pll[4], o_O=self.cd_sys2x.clk)
        self.specials += Instance("BUFG", i_I=pll[5], o_O=self.cd_sys.clk)
        reset = ~platform.request("cpu_reset") | self.reset
        self.clock_domains.cd_por = ClockDomain()
        por = Signal(max=1 << 11, reset=(1 << 11) - 1)
        self.sync.por += If(por != 0, por.eq(por - 1))
        self.comb += self.cd_por.clk.eq(self.cd_sys.clk)
        self.specials += AsyncResetSynchronizer(self.cd_por, reset)
        self.specials += AsyncResetSynchronizer(self.cd_sys2x, ~pll_lckd | (por > 0))
        self.specials += AsyncResetSynchronizer(self.cd_sys, ~pll_lckd | (por > 0))
        self.specials += Instance("BUFG", i_I=pll[2], o_O=self.cd_sdram_half.clk)
        self.specials += Instance("BUFPLL", p_DIVIDE=4,
                                  i_PLLIN=pll[0], i_GCLK=self.cd_sys2x.clk,
                                  i_LOCKED=pll_lckd, o_IOCLK=self.cd_sdram_full_wr.clk,
                                  o_SERDESSTROBE=self.clk8x_wr_strb)
        self.comb += [
            self.cd_sdram_full_rd.clk.eq(self.cd_sdram_full_wr.clk),
            self.clk8x_rd_strb.eq(self.clk8x_wr_strb),
        ]
        clk_sdram_half_shifted = Signal()
        self.specials += Instance("BUFG", i_I=pll[3], o_O=clk_sdram_half_shifted)

        output_clk = Signal()
        clk = platform.request("ddram_clock")
        self.specials += Instance("ODDR2", p_DDR_ALIGNMENT="NONE",
                                  p_INIT=0, p_SRTYPE="SYNC",
                                  i_D0=1, i_D1=0, i_S=0, i_R=0, i_CE=1,
                                  i_C0=clk_sdram_half_shifted, i_C1=~clk_sdram_half_shifted,
                                  o_Q=output_clk)
        self.specials += Instance("OBUFDS", i_I=output_clk, o_O=clk.p, o_OB=clk.n)

        dcm_base50_locked = Signal()
        self.specials += Instance("DCM_CLKGEN",
                                  p_CLKFXDV_DIVIDE=2, p_CLKFX_DIVIDE=4, p_CLKFX_MD_MAX=1.0, p_CLKFX_MULTIPLY=2,
                                  p_CLKIN_PERIOD=10.0, p_SPREAD_SPECTRUM="NONE", p_STARTUP_WAIT="FALSE",

                                  i_CLKIN=clk100a, o_CLKFX=self.cd_base50.clk,
                                  o_LOCKED=dcm_base50_locked,
                                  i_FREEZEDCM=0, i_RST=ResetSignal())
        self.specials += AsyncResetSynchronizer(self.cd_base50, self.cd_sys.rst | ~dcm_base50_locked)
        platform.add_period_constraint(self.cd_base50.clk, 20)

class BaseSoC(SoCSDRAM):
    csr_peripherals = (
        "front_panel",
        "ddrphy",
        "dna",
    )
    csr_map_update(SoCSDRAM.csr_map, csr_peripherals)

    mem_map = {
        "firmware_ram": 0x20000000,  # (default shadow @0xa0000000)
    }
    mem_map.update(SoCSDRAM.mem_map)

    def __init__(self, platform,
                 firmware_ram_size=0x10000,
                 firmware_filename="firmware/firmware.bin",
                 **kwargs):
        clk_freq = 50*1000000
        SoCSDRAM.__init__(self, platform, clk_freq,
            integrated_rom_size=0x8000,
            integrated_sram_size=0x8000,
            **kwargs)
        self.submodules.crg = _CRG(platform, clk_freq)
        self.submodules.dna = dna.DNA()

        # front panel (ATX)
        self.submodules.front_panel = FrontPanelGPIO(platform, clk_freq)
        self.comb += self.crg.reset.eq(self.front_panel.reset)

        # firmware
        self.submodules.firmware_ram = firmware.FirmwareROM(firmware_ram_size, firmware_filename)
        self.register_mem("firmware_ram", self.mem_map["firmware_ram"], self.firmware_ram.bus, firmware_ram_size)
        self.add_constant("ROM_BOOT_ADDRESS", self.mem_map["firmware_ram"])

        # sdram
        self.submodules.ddrphy = s6ddrphy.S6QuarterRateDDRPHY(platform.request("ddram"),
                                                              rd_bitslip=0,
                                                              wr_bitslip=4,
                                                              dqs_ddr_alignment="C0")
        sdram_module = MT41J128M16(self.clk_freq)
        self.register_sdram(self.ddrphy, "lasmicon",
                            sdram_module.geom_settings,
                            sdram_module.timing_settings,
                            controller_settings=ControllerSettings(with_bandwidth=True))
        self.comb += [
            self.ddrphy.clk8x_wr_strb.eq(self.crg.clk8x_wr_strb),
            self.ddrphy.clk8x_rd_strb.eq(self.crg.clk8x_rd_strb),
        ]

        self.platform.add_period_constraint(self.crg.cd_sys.clk, 1/clk_freq*1e9)

class MiniSoC(BaseSoC):
    csr_peripherals = (
        "ethphy",
        "ethmac",
        "analyzer"
    )
    csr_map_update(SoCSDRAM.csr_map, csr_peripherals)

    interrupt_map = {
        "ethmac": 2,
    }
    interrupt_map.update(BaseSoC.interrupt_map)

    mem_map = {
        "ethmac": 0x30000000,  # (shadow @0xb0000000)
    }
    mem_map.update(BaseSoC.mem_map)

    def __init__(self, *args, **kwargs):
        BaseSoC.__init__(self, *args, **kwargs)

        self.submodules.ethphy = LiteEthPHYRGMII(self.platform.request("eth_clocks"),
                                                 self.platform.request("eth"))
        self.submodules.ethmac = LiteEthMAC(phy=self.ethphy, dw=32, interface="wishbone")
        self.add_wb_slave(mem_decoder(self.mem_map["ethmac"]), self.ethmac.bus)
        self.add_memory_region("ethmac", self.mem_map["ethmac"] | self.shadow_base, 0x2000)

        self.submodules.bridge = UARTWishboneBridge(self.platform.request("serial_debug"), self.clk_freq, baudrate=115200)
        self.add_wb_master(self.bridge.wishbone)

        self.specials += [
            Keep(self.ethphy.crg.cd_eth_rx.clk),
            Keep(self.ethphy.crg.cd_eth_tx.clk)
        ]

        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_rx.clk, 8.0)
        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_tx.clk, 8.0)

        self.platform.add_false_path_constraints(
            self.crg.cd_sys.clk,
            self.ethphy.crg.cd_eth_rx.clk,
            self.ethphy.crg.cd_eth_tx.clk)

        self.platform.add_platform_command("""
NET "{eth_clocks_rx}" CLOCK_DEDICATED_ROUTE = FALSE;
PIN "BUFG_5.O" CLOCK_DEDICATED_ROUTE = FALSE;
""", eth_clocks_rx=self.platform.lookup_request("eth_clocks").rx)

def main():
    parser = argparse.ArgumentParser(description="Opsis LiteX SoC")
    builder_args(parser)
    soc_sdram_args(parser)
    parser.add_argument("--with-ethernet", action="store_true",
                        help="enable Ethernet support")
    parser.add_argument("--nocompile-gateware", action="store_true")
    args = parser.parse_args()

    platform = opsis_platform.Platform()
    cls = MiniSoC if args.with_ethernet else BaseSoC
    soc = cls(platform, **soc_sdram_argdict(args))
    builder = Builder(soc, output_dir="build",
                      compile_gateware=not args.nocompile_gateware,
                      csr_csv="test/csr.csv")
    vns = builder.build()

if __name__ == "__main__":
    main()
<commit_msg>opsis_base: use new sdram module from LiteX<commit_after>#!/usr/bin/env python3

import argparse
import os
import struct
from fractions import Fraction
import importlib

from litex.gen import *
from litex.gen.fhdl.specials import Keep
from litex.gen.genlib.io import CRG
from litex.gen.genlib.resetsync import AsyncResetSynchronizer
from litex.gen.genlib.misc import WaitTimer

from litex.soc.cores.sdram.settings import MT41J128M16
from litex.soc.cores.sdram.phy import s6ddrphy
from litex.soc.integration.soc_core import mem_decoder
from litex.soc.integration.soc_sdram import *
from litex.soc.integration.builder import *
from litex.soc.cores.gpio import GPIOIn, GPIOOut
from litex.soc.interconnect.csr import AutoCSR
from litex.soc.cores.uart.bridge import UARTWishboneBridge
from litex.soc.cores.sdram.lasmicon.core import ControllerSettings

from liteeth.phy.s6rgmii import LiteEthPHYRGMII
from liteeth.core.mac import LiteEthMAC

import opsis_platform

from gateware import dna
from gateware import firmware

def csr_map_update(csr_map, csr_peripherals):
  csr_map.update(dict((n, v) for v, n in enumerate(csr_peripherals, start=max(csr_map.values()) + 1)))

class FrontPanelGPIO(Module, AutoCSR):
    def __init__(self, platform, clk_freq):
        switches = Signal(1)
        leds = Signal(2)

        self.reset = Signal()

        # # #

        self.submodules.switches = GPIOIn(switches)
        self.submodules.leds = GPIOOut(leds)
        self.comb += [
           switches[0].eq(~platform.request("pwrsw")),
           platform.request("hdled").eq(~leds[0]),
           platform.request("pwled").eq(~leds[1]),
        ]

        # generate a reset when power switch is pressed for 1 second
        self.submodules.reset_timer = WaitTimer(clk_freq)
        self.comb += [
            self.reset_timer.wait.eq(switches[0]),
            self.reset.eq(self.reset_timer.done)
        ]

class _CRG(Module):
    def __init__(self, platform, clk_freq):
        self.clock_domains.cd_sys = ClockDomain()
        self.clock_domains.cd_sys2x = ClockDomain()
        self.clock_domains.cd_sdram_half = ClockDomain()
        self.clock_domains.cd_sdram_full_wr = ClockDomain()
        self.clock_domains.cd_sdram_full_rd = ClockDomain()
        self.clock_domains.cd_base50 = ClockDomain()

        self.clk8x_wr_strb = Signal()
        self.clk8x_rd_strb = Signal()

        self.reset = Signal()

        f0 = 100*1000000
        clk100 = platform.request("clk100")
        clk100a = Signal()
        self.specials += Instance("IBUFG", i_I=clk100, o_O=clk100a)
        clk100b = Signal()
        self.specials += Instance("BUFIO2", p_DIVIDE=1,
                                  p_DIVIDE_BYPASS="TRUE", p_I_INVERT="FALSE",
                                  i_I=clk100a, o_DIVCLK=clk100b)
        f = Fraction(int(clk_freq), int(f0))
        n, m = f.denominator, f.numerator
        assert f0/n*m == clk_freq
        p = 8
        pll_lckd = Signal()
        pll_fb = Signal()
        pll = Signal(6)
        self.specials.pll = Instance("PLL_ADV", p_SIM_DEVICE="SPARTAN6",
                                     p_BANDWIDTH="OPTIMIZED", p_COMPENSATION="INTERNAL",
                                     p_REF_JITTER=.01, p_CLK_FEEDBACK="CLKFBOUT",
                                     i_DADDR=0, i_DCLK=0, i_DEN=0, i_DI=0, i_DWE=0, i_RST=0, i_REL=0,
                                     p_DIVCLK_DIVIDE=1, p_CLKFBOUT_MULT=m*p//n, p_CLKFBOUT_PHASE=0.,
                                     i_CLKIN1=clk100b, i_CLKIN2=0, i_CLKINSEL=1,
                                     p_CLKIN1_PERIOD=1e9/f0, p_CLKIN2_PERIOD=0.,
                                     i_CLKFBIN=pll_fb, o_CLKFBOUT=pll_fb, o_LOCKED=pll_lckd,
                                     o_CLKOUT0=pll[0], p_CLKOUT0_DUTY_CYCLE=.5,
                                     o_CLKOUT1=pll[1], p_CLKOUT1_DUTY_CYCLE=.5,
                                     o_CLKOUT2=pll[2], p_CLKOUT2_DUTY_CYCLE=.5,
                                     o_CLKOUT3=pll[3], p_CLKOUT3_DUTY_CYCLE=.5,
                                     o_CLKOUT4=pll[4], p_CLKOUT4_DUTY_CYCLE=.5,
                                     o_CLKOUT5=pll[5], p_CLKOUT5_DUTY_CYCLE=.5,
                                     p_CLKOUT0_PHASE=0., p_CLKOUT0_DIVIDE=p//8,    # sdram wr/rd full clock
                                     p_CLKOUT1_PHASE=0., p_CLKOUT1_DIVIDE=p//8,    # unused
                                     p_CLKOUT2_PHASE=230., p_CLKOUT2_DIVIDE=p//4,  # sdram dqs adr ctrl off-chip
                                     p_CLKOUT3_PHASE=210., p_CLKOUT3_DIVIDE=p//4,  # ddr half clock
                                     p_CLKOUT4_PHASE=0., p_CLKOUT4_DIVIDE=p//2,    # 2x system clock
                                     p_CLKOUT5_PHASE=0., p_CLKOUT5_DIVIDE=p//1,    # system clock
        )
        self.specials += Instance("BUFG", i_I=pll[4], o_O=self.cd_sys2x.clk)
        self.specials += Instance("BUFG", i_I=pll[5], o_O=self.cd_sys.clk)
        reset = ~platform.request("cpu_reset") | self.reset
        self.clock_domains.cd_por = ClockDomain()
        por = Signal(max=1 << 11, reset=(1 << 11) - 1)
        self.sync.por += If(por != 0, por.eq(por - 1))
        self.comb += self.cd_por.clk.eq(self.cd_sys.clk)
        self.specials += AsyncResetSynchronizer(self.cd_por, reset)
        self.specials += AsyncResetSynchronizer(self.cd_sys2x, ~pll_lckd | (por > 0))
        self.specials += AsyncResetSynchronizer(self.cd_sys, ~pll_lckd | (por > 0))
        self.specials += Instance("BUFG", i_I=pll[2], o_O=self.cd_sdram_half.clk)
        self.specials += Instance("BUFPLL", p_DIVIDE=4,
                                  i_PLLIN=pll[0], i_GCLK=self.cd_sys2x.clk,
                                  i_LOCKED=pll_lckd, o_IOCLK=self.cd_sdram_full_wr.clk,
                                  o_SERDESSTROBE=self.clk8x_wr_strb)
        self.comb += [
            self.cd_sdram_full_rd.clk.eq(self.cd_sdram_full_wr.clk),
            self.clk8x_rd_strb.eq(self.clk8x_wr_strb),
        ]
        clk_sdram_half_shifted = Signal()
        self.specials += Instance("BUFG", i_I=pll[3], o_O=clk_sdram_half_shifted)

        output_clk = Signal()
        clk = platform.request("ddram_clock")
        self.specials += Instance("ODDR2", p_DDR_ALIGNMENT="NONE",
                                  p_INIT=0, p_SRTYPE="SYNC",
                                  i_D0=1, i_D1=0, i_S=0, i_R=0, i_CE=1,
                                  i_C0=clk_sdram_half_shifted, i_C1=~clk_sdram_half_shifted,
                                  o_Q=output_clk)
        self.specials += Instance("OBUFDS", i_I=output_clk, o_O=clk.p, o_OB=clk.n)

        dcm_base50_locked = Signal()
        self.specials += Instance("DCM_CLKGEN",
                                  p_CLKFXDV_DIVIDE=2, p_CLKFX_DIVIDE=4, p_CLKFX_MD_MAX=1.0, p_CLKFX_MULTIPLY=2,
                                  p_CLKIN_PERIOD=10.0, p_SPREAD_SPECTRUM="NONE", p_STARTUP_WAIT="FALSE",

                                  i_CLKIN=clk100a, o_CLKFX=self.cd_base50.clk,
                                  o_LOCKED=dcm_base50_locked,
                                  i_FREEZEDCM=0, i_RST=ResetSignal())
        self.specials += AsyncResetSynchronizer(self.cd_base50, self.cd_sys.rst | ~dcm_base50_locked)
        platform.add_period_constraint(self.cd_base50.clk, 20)

class BaseSoC(SoCSDRAM):
    csr_peripherals = (
        "front_panel",
        "ddrphy",
        "dna",
    )
    csr_map_update(SoCSDRAM.csr_map, csr_peripherals)

    mem_map = {
        "firmware_ram": 0x20000000,  # (default shadow @0xa0000000)
    }
    mem_map.update(SoCSDRAM.mem_map)

    def __init__(self, platform,
                 firmware_ram_size=0x10000,
                 firmware_filename="firmware/firmware.bin",
                 **kwargs):
        clk_freq = 50*1000000
        SoCSDRAM.__init__(self, platform, clk_freq,
            integrated_rom_size=0x8000,
            integrated_sram_size=0x8000,
            **kwargs)
        self.submodules.crg = _CRG(platform, clk_freq)
        self.submodules.dna = dna.DNA()

        # front panel (ATX)
        self.submodules.front_panel = FrontPanelGPIO(platform, clk_freq)
        self.comb += self.crg.reset.eq(self.front_panel.reset)

        # firmware
        self.submodules.firmware_ram = firmware.FirmwareROM(firmware_ram_size, firmware_filename)
        self.register_mem("firmware_ram", self.mem_map["firmware_ram"], self.firmware_ram.bus, firmware_ram_size)
        self.add_constant("ROM_BOOT_ADDRESS", self.mem_map["firmware_ram"])

        # sdram
        self.submodules.ddrphy = s6ddrphy.S6QuarterRateDDRPHY(platform.request("ddram"),
                                                              rd_bitslip=0,
                                                              wr_bitslip=4,
                                                              dqs_ddr_alignment="C0")
        sdram_module = MT41J128M16(self.clk_freq, "1:4")
        self.register_sdram(self.ddrphy, "lasmicon",
                            sdram_module.geom_settings,
                            sdram_module.timing_settings,
                            controller_settings=ControllerSettings(with_bandwidth=True))
        self.comb += [
            self.ddrphy.clk8x_wr_strb.eq(self.crg.clk8x_wr_strb),
            self.ddrphy.clk8x_rd_strb.eq(self.crg.clk8x_rd_strb),
        ]

        self.platform.add_period_constraint(self.crg.cd_sys.clk, 1/clk_freq*1e9)

class MiniSoC(BaseSoC):
    csr_peripherals = (
        "ethphy",
        "ethmac",
        "analyzer"
    )
    csr_map_update(SoCSDRAM.csr_map, csr_peripherals)

    interrupt_map = {
        "ethmac": 2,
    }
    interrupt_map.update(BaseSoC.interrupt_map)

    mem_map = {
        "ethmac": 0x30000000,  # (shadow @0xb0000000)
    }
    mem_map.update(BaseSoC.mem_map)

    def __init__(self, *args, **kwargs):
        BaseSoC.__init__(self, *args, **kwargs)

        self.submodules.ethphy = LiteEthPHYRGMII(self.platform.request("eth_clocks"),
                                                 self.platform.request("eth"))
        self.submodules.ethmac = LiteEthMAC(phy=self.ethphy, dw=32, interface="wishbone")
        self.add_wb_slave(mem_decoder(self.mem_map["ethmac"]), self.ethmac.bus)
        self.add_memory_region("ethmac", self.mem_map["ethmac"] | self.shadow_base, 0x2000)

        self.submodules.bridge = UARTWishboneBridge(self.platform.request("serial_debug"), self.clk_freq, baudrate=115200)
        self.add_wb_master(self.bridge.wishbone)

        self.specials += [
            Keep(self.ethphy.crg.cd_eth_rx.clk),
            Keep(self.ethphy.crg.cd_eth_tx.clk)
        ]

        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_rx.clk, 8.0)
        self.platform.add_period_constraint(self.ethphy.crg.cd_eth_tx.clk, 8.0)

        self.platform.add_false_path_constraints(
            self.crg.cd_sys.clk,
            self.ethphy.crg.cd_eth_rx.clk,
            self.ethphy.crg.cd_eth_tx.clk)

        self.platform.add_platform_command("""
NET "{eth_clocks_rx}" CLOCK_DEDICATED_ROUTE = FALSE;
PIN "BUFG_5.O" CLOCK_DEDICATED_ROUTE = FALSE;
""", eth_clocks_rx=self.platform.lookup_request("eth_clocks").rx)

def main():
    parser = argparse.ArgumentParser(description="Opsis LiteX SoC")
    builder_args(parser)
    soc_sdram_args(parser)
    parser.add_argument("--with-ethernet", action="store_true",
                        help="enable Ethernet support")
    parser.add_argument("--nocompile-gateware", action="store_true")
    args = parser.parse_args()

    platform = opsis_platform.Platform()
    cls = MiniSoC if args.with_ethernet else BaseSoC
    soc = cls(platform, **soc_sdram_argdict(args))
    builder = Builder(soc, output_dir="build",
                      compile_gateware=not args.nocompile_gateware,
                      csr_csv="test/csr.csv")
    vns = builder.build()

if __name__ == "__main__":
    main()
<|endoftext|>