<commit_before># coding: utf-8

""" Class for reading data from SCF simulations """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import logging
import re

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G
from astropy.table import Table

# Project
from .core import NBodyReader

# Create logger
logger = logging.getLogger(__name__)

__all__ = ["SCFReader"]

class SCFReader(NBodyReader):

    # TODO: way to list snapshot files?

    def _read_units(self):
        """ Read and parse the SCFPAR file containing simulation parameters
            and initial conditions. Right now, only parse out the simulation
            units.
        """
        pars = dict()

        parfile = os.path.join(self.path, "SCFPAR")
        with open(parfile) as f:
            lines = f.readlines()

            # find what G is set to
            pars['G'] = float(lines[6].split()[0])
            pars['length'] = float(lines[16].split()[0])
            pars['mass'] = float(lines[17].split()[0])

        _G = G.decompose(bases=[u.kpc,u.M_sun,u.Myr]).value
        X = (_G / pars['length']**3 * pars['mass'])**-0.5

        length_unit = u.Unit("{0} kpc".format(pars['length']))
        mass_unit = u.Unit("{0} M_sun".format(pars['mass']))
        time_unit = u.Unit("{:08f} Myr".format(X))

        units = dict(length=length_unit,
                     mass=mass_unit,
                     time=time_unit,
                     speed=length_unit/time_unit,
                     dimensionless=u.dimensionless_unscaled)

        return units

    def read_snap(self, filename, units=None):
        """ Given a SNAP filename, read and return the data. By default,
            returns data in simulation units, but this can be changed with
            the `units` kwarg.

            Parameters
            ----------
            filename : str
                The name of the SNAP file to read.
            units : dict (optional)
                A unit system to transform the data to. If None, will return
                the data in simulation units.
        """

        # read the first line to get the numer of particles and timestep
        fullpath = os.path.join(self.path, filename)
        with open(fullpath) as f:
            firstline = f.readline()
            try:
                nparticles,time = firstline.split()
            except:
                raise ValueError("Invalid header line. Expected 'nparticles,time', "
                                 "got:\n\t\t{}".format(firstline))
            numcols = len(f.readline().split())

        time = float(time)*self.sim_units['time']

        if numcols == 8:
            # not self gravitating
            logger.debug("Not a self-gravitating run: only 8 columns")

            # column names for SNAP file, in simulation units
            colnames = "m x y z vx vy vz s1".split()
            coltypes = "mass length length length speed speed speed dimensionless".split()
            colunits = [self.sim_units[x] for x in coltypes]

        elif numcols == 10:
            # not self gravitating
            logger.debug("A self-gravitating run: 10 columns")

            # column names for SNAP file, in simulation units
            colnames = "m x y z vx vy vz s1 s2 tub".split()
            coltypes = "mass length length length speed speed speed dimensionless dimensionless time".split()
            colunits = [self.sim_units[x] for x in coltypes]

        else:
            raise ValueError("Invalid SNAP file: {} columns (not 8 or 10).".format(numcols))

        data = np.genfromtxt(fullpath, skiprows=1, names=colnames)
        if units is not None:
            new_colunits = []
            for colname,colunit in zip(colnames,colunits):
                newdata = (data[colname]*colunit).decompose(units)
                data[colname] = newdata.value
                new_colunits.append(newdata.unit)

            time = time.decompose(units)
            colunits = new_colunits

        tbl = Table(data, meta=dict(time=time.value))
        for colname,colunit in zip(colnames,colunits):
            tbl[colname].unit = colunit

        return tbl

    def read_cen(self, units=None):
        """ Read the SCFCEN file data. By default, returns data in simulation
            units, but this can be changed with the `units` kwarg.

            Parameters
            ----------
            units : dict (optional)
                A unit system to transform the data to. If None, will return
                the data in simulation units.
        """
        fullpath = os.path.join(self.path, "SCFCEN")

        # column names for SNAP file, in simulation units
        colnames = "t dt x y z vx vy vz".split()
        coltypes = "time time length length length speed speed speed".split()
        colunits = [self.sim_units[x] for x in coltypes]

        data = np.genfromtxt(fullpath, skiprows=1, names=colnames)
        if units is not None:
            new_colunits = []
            for colname,colunit in zip(colnames,colunits):
                newdata = (data[colname]*colunit).decompose(units)
                data[colname] = newdata.value
                new_colunits.append(newdata.unit)

            colunits = new_colunits

        tbl = Table(data)
        for colname,colunit in zip(colnames,colunits):
            tbl[colname].unit = colunit

        return tbl
<commit_msg>more flexible reader<commit_after># coding: utf-8

""" Class for reading data from SCF simulations """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import logging
import re

# Third-party
import numpy as np
import astropy.units as u
from astropy.constants import G
from astropy.table import Table

# Project
from .core import NBodyReader

# Create logger
logger = logging.getLogger(__name__)

__all__ = ["SCFReader"]

class SCFReader(NBodyReader):

    # TODO: way to list snapshot files?

    def _read_units(self):
        """ Read and parse the SCFPAR file containing simulation parameters
            and initial conditions. Right now, only parse out the simulation
            units.
        """
        pars = dict()

        parfile = os.path.join(self.path, "SCFPAR")
        with open(parfile) as f:
            lines = f.readlines()

            # find what G is set to
            for i,line in enumerate(line):
                if line.split()[1].strip() == "G":
                    break
            pars['G'] = float(lines[i].split()[0])
            pars['length'] = float(lines[i+10].split()[0])
            pars['mass'] = float(lines[i+11].split()[0])

        _G = G.decompose(bases=[u.kpc,u.M_sun,u.Myr]).value
        X = (_G / pars['length']**3 * pars['mass'])**-0.5

        length_unit = u.Unit("{0} kpc".format(pars['length']))
        mass_unit = u.Unit("{0} M_sun".format(pars['mass']))
        time_unit = u.Unit("{:08f} Myr".format(X))

        units = dict(length=length_unit,
                     mass=mass_unit,
                     time=time_unit,
                     speed=length_unit/time_unit,
                     dimensionless=u.dimensionless_unscaled)

        return units

    def read_snap(self, filename, units=None):
        """ Given a SNAP filename, read and return the data. By default,
            returns data in simulation units, but this can be changed with
            the `units` kwarg.

            Parameters
            ----------
            filename : str
                The name of the SNAP file to read.
            units : dict (optional)
                A unit system to transform the data to. If None, will return
                the data in simulation units.
        """

        # read the first line to get the numer of particles and timestep
        fullpath = os.path.join(self.path, filename)
        with open(fullpath) as f:
            firstline = f.readline()
            try:
                nparticles,time = firstline.split()
            except:
                raise ValueError("Invalid header line. Expected 'nparticles,time', "
                                 "got:\n\t\t{}".format(firstline))
            numcols = len(f.readline().split())

        time = float(time)*self.sim_units['time']

        if numcols == 8:
            # not self gravitating
            logger.debug("Not a self-gravitating run: only 8 columns")

            # column names for SNAP file, in simulation units
            colnames = "m x y z vx vy vz s1".split()
            coltypes = "mass length length length speed speed speed dimensionless".split()
            colunits = [self.sim_units[x] for x in coltypes]

        elif numcols == 10:
            # not self gravitating
            logger.debug("A self-gravitating run: 10 columns")

            # column names for SNAP file, in simulation units
            colnames = "m x y z vx vy vz s1 s2 tub".split()
            coltypes = "mass length length length speed speed speed dimensionless dimensionless time".split()
            colunits = [self.sim_units[x] for x in coltypes]

        else:
            raise ValueError("Invalid SNAP file: {} columns (not 8 or 10).".format(numcols))

        data = np.genfromtxt(fullpath, skiprows=1, names=colnames)
        if units is not None:
            new_colunits = []
            for colname,colunit in zip(colnames,colunits):
                newdata = (data[colname]*colunit).decompose(units)
                data[colname] = newdata.value
                new_colunits.append(newdata.unit)

            time = time.decompose(units)
            colunits = new_colunits

        tbl = Table(data, meta=dict(time=time.value))
        for colname,colunit in zip(colnames,colunits):
            tbl[colname].unit = colunit

        return tbl

    def read_cen(self, units=None):
        """ Read the SCFCEN file data. By default, returns data in simulation
            units, but this can be changed with the `units` kwarg.

            Parameters
            ----------
            units : dict (optional)
                A unit system to transform the data to. If None, will return
                the data in simulation units.
        """
        fullpath = os.path.join(self.path, "SCFCEN")

        # column names for SNAP file, in simulation units
        colnames = "t dt x y z vx vy vz".split()
        coltypes = "time time length length length speed speed speed".split()
        colunits = [self.sim_units[x] for x in coltypes]

        data = np.genfromtxt(fullpath, skiprows=1, names=colnames)
        if units is not None:
            new_colunits = []
            for colname,colunit in zip(colnames,colunits):
                newdata = (data[colname]*colunit).decompose(units)
                data[colname] = newdata.value
                new_colunits.append(newdata.unit)

            colunits = new_colunits

        tbl = Table(data)
        for colname,colunit in zip(colnames,colunits):
            tbl[colname].unit = colunit

        return tbl
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

from pybossa.model import db
from pybossa.model.app import App
from pybossa.model.blogpost import Blogpost
from pybossa.model.category import Category
from pybossa.model.featured import Featured
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.user import User

import factory
from factory.alchemy import SQLAlchemyModelFactory

def reset_all_pk_sequences():
    AppFactory.reset_sequence()
    BlogpostFactory.reset_sequence()
    CategoryFactory.reset_sequence()
    FeaturedFactory.reset_sequence()
    TaskFactory.reset_sequence()
    TaskRunFactory.reset_sequence()
    UserFactory.reset_sequence()

class SQLAlchemyPyBossaFactory(SQLAlchemyModelFactory):
    FACTORY_SESSION = db.session

    @classmethod
    def _create(cls, target_class, *args, **kwargs):
        """The default beahaviour is to simply add the object to the SQLAlchemy
        session. Here, we also flush it as autoflush is disabled in the
        flask-SQLAlchemy extension"""
        session = cls.FACTORY_SESSION
        obj = target_class(*args, **kwargs)
        session.add(obj)
        session.commit()
        return obj

class AppFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = App

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'My App number %d' % n)
    short_name = factory.Sequence(lambda n: u'app%d' % n)
    description = u'App description'
    allow_anonymous_contributors = True
    long_tasks = 0
    hidden = 0
    owner = factory.SubFactory('factories.UserFactory')
    owner_id = factory.LazyAttribute(lambda app: app.owner.id)
    category = factory.SubFactory('factories.CategoryFactory')
    category_id = factory.LazyAttribute(lambda app: app.category.id)

class BlogpostFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Blogpost

    id = factory.Sequence(lambda n: n)
    title = u'Blogpost title'
    body = u'Blogpost body text'
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda blogpost: blogpost.app.id)
    owner = factory.SelfAttribute('app.owner')
    user_id = factory.LazyAttribute(lambda blogpost: blogpost.owner.id)

class CategoryFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Category

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: 'category_name_%d' % n)
    short_name = factory.Sequence(lambda n: 'category_short_name_%d' % n)
    description = 'Category description for testing purposes'

class FeaturedFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Featured

    id = factory.Sequence(lambda n: n)
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda featured: featured.app.id)

class TaskFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Task

    id = factory.Sequence(lambda n: n)
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda task: task.app.id)
    state = u'ongoing'
    quorum = 0
    calibration = 0
    priority_0 = 0.0
    n_answers = 30

class TaskRunFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = TaskRun

    id = factory.Sequence(lambda n: n)
    task = factory.SubFactory('factories.TaskFactory')
    task_id = factory.LazyAttribute(lambda task_run: task_run.task.id)
    app = factory.SelfAttribute('task.app')
    app_id = factory.LazyAttribute(lambda task_run: task_run.app.id)
    user = factory.SubFactory('factories.UserFactory')
    user_id = factory.LazyAttribute(lambda task_run: task_run.user.id)
    user_ip = None

class UserFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = User

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'user%d' % n)
    fullname = factory.Sequence(lambda n: u'User %d' % n)
    email_addr = factory.LazyAttribute(lambda usr: u'%s@test.com' % usr.name)
    locale = u'en'
    admin = False
    privacy_mode = True
    api_key =  factory.Sequence(lambda n: u'api-key%d' % n)
<commit_msg>Created AnonymousTaskRun factory<commit_after># -*- coding: utf8 -*-
# This file is part of PyBossa.
#
# Copyright (C) 2013 SF Isle of Man Limited
#
# PyBossa is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyBossa is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with PyBossa.  If not, see <http://www.gnu.org/licenses/>.

from pybossa.model import db
from pybossa.model.app import App
from pybossa.model.blogpost import Blogpost
from pybossa.model.category import Category
from pybossa.model.featured import Featured
from pybossa.model.task import Task
from pybossa.model.task_run import TaskRun
from pybossa.model.user import User

import factory
from factory.alchemy import SQLAlchemyModelFactory

def reset_all_pk_sequences():
    AppFactory.reset_sequence()
    BlogpostFactory.reset_sequence()
    CategoryFactory.reset_sequence()
    FeaturedFactory.reset_sequence()
    TaskFactory.reset_sequence()
    TaskRunFactory.reset_sequence()
    UserFactory.reset_sequence()

class SQLAlchemyPyBossaFactory(SQLAlchemyModelFactory):
    FACTORY_SESSION = db.session

    @classmethod
    def _create(cls, target_class, *args, **kwargs):
        """The default beahaviour is to simply add the object to the SQLAlchemy
        session. Here, we also flush it as autoflush is disabled in the
        flask-SQLAlchemy extension"""
        session = cls.FACTORY_SESSION
        obj = target_class(*args, **kwargs)
        session.add(obj)
        session.commit()
        return obj

class AppFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = App

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'My App number %d' % n)
    short_name = factory.Sequence(lambda n: u'app%d' % n)
    description = u'App description'
    allow_anonymous_contributors = True
    long_tasks = 0
    hidden = 0
    owner = factory.SubFactory('factories.UserFactory')
    owner_id = factory.LazyAttribute(lambda app: app.owner.id)
    category = factory.SubFactory('factories.CategoryFactory')
    category_id = factory.LazyAttribute(lambda app: app.category.id)

class BlogpostFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Blogpost

    id = factory.Sequence(lambda n: n)
    title = u'Blogpost title'
    body = u'Blogpost body text'
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda blogpost: blogpost.app.id)
    owner = factory.SelfAttribute('app.owner')
    user_id = factory.LazyAttribute(lambda blogpost: blogpost.owner.id)

class CategoryFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Category

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: 'category_name_%d' % n)
    short_name = factory.Sequence(lambda n: 'category_short_name_%d' % n)
    description = 'Category description for testing purposes'

class FeaturedFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Featured

    id = factory.Sequence(lambda n: n)
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda featured: featured.app.id)

class TaskFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = Task

    id = factory.Sequence(lambda n: n)
    app = factory.SubFactory('factories.AppFactory')
    app_id = factory.LazyAttribute(lambda task: task.app.id)
    state = u'ongoing'
    quorum = 0
    calibration = 0
    priority_0 = 0.0
    n_answers = 30

class TaskRunFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = TaskRun

    id = factory.Sequence(lambda n: n)
    task = factory.SubFactory('factories.TaskFactory')
    task_id = factory.LazyAttribute(lambda task_run: task_run.task.id)
    app = factory.SelfAttribute('task.app')
    app_id = factory.LazyAttribute(lambda task_run: task_run.app.id)
    user = factory.SubFactory('factories.UserFactory')
    user_id = factory.LazyAttribute(lambda task_run: task_run.user.id)

class AnonymousTaskRunFactory(TaskRunFactory):
    user = None
    user_id = None
    user_ip = '127.0.0.1'

class UserFactory(SQLAlchemyPyBossaFactory):
    FACTORY_FOR = User

    id = factory.Sequence(lambda n: n)
    name = factory.Sequence(lambda n: u'user%d' % n)
    fullname = factory.Sequence(lambda n: u'User %d' % n)
    email_addr = factory.LazyAttribute(lambda usr: u'%s@test.com' % usr.name)
    locale = u'en'
    admin = False
    privacy_mode = True
    api_key =  factory.Sequence(lambda n: u'api-key%d' % n)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" Merges the debug symbols and uploads them to cipd.
"""

import argparse
import collections
import json
import os
import platform
import shutil
import subprocess
import sys
import tarfile

def IsLinux():
  return platform.system() == 'Linux'

def GetPackagingDir(out_dir):
  return os.path.abspath(os.path.join(out_dir, os.pardir))

def CreateCIPDDefinition(target_arch, out_dir):
  dir_name = os.path.basename(os.path.normpath(out_dir))
  return """
package: flutter/fuchsia-debug-symbols-%s
description: Flutter and Dart runner debug symbols for Fuchsia. Target architecture %s.
install_mode: copy
data:
  - dir: %s
""" % (target_arch, target_arch, dir_name)

# CIPD CLI needs the definition and data directory to be relative to each other.
def WriteCIPDDefinition(target_arch, out_dir):
  _packaging_dir = GetPackagingDir(out_dir)
  yaml_file = os.path.join(_packaging_dir, 'debug_symbols.cipd.yaml')
  with open(yaml_file, 'w') as f:
    cipd_def = CreateCIPDDefinition(target_arch, out_dir)
    f.write(cipd_def)
  return yaml_file

def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
  _packaging_dir = GetPackagingDir(out_dir)
  if upload and IsLinux():
    command = [
        'cipd', 'create', '-pkg-def', cipd_yaml, '-ref', 'latest', '-tag',
        'git_revision:%s' % engine_version
    ]
  else:
    command = [
        'cipd', 'pkg-build', '-pkg-def', cipd_yaml, '-out',
        os.path.join(_packaging_dir,
                     'fuchsia-debug-symbols-%s.cipd' % target_arch)
    ]

  # Retry up to three times.  We've seen CIPD fail on verification in some
  # instances. Normally verification takes slightly more than 1 minute when
  # it succeeds.
  num_tries = 3
  for tries in range(num_tries):
    try:
      subprocess.check_call(command, cwd=_packaging_dir)
      break
    except subprocess.CalledProcessError:
      print('Failed %s times' % tries + 1)
      if tries == num_tries - 1:
        raise

# Recursively hardlinks contents from one directory to another,
# skipping over collisions.
def HardlinkContents(dirA, dirB):
  for src_dir, _, filenames in os.walk(dirA):
    for filename in filenames:
      src = os.path.join(src_dir, filename)
      dest_dir = os.path.join(dirB, os.path.relpath(src_dir, dirA))
      os.makedirs(dest_dir)
      dest = os.path.join(dest_dir, filename)
      if os.path.exists(dest):
        # The last two path components provide a content address for a .build-id entry.
        tokens = os.path.split(dest)
        name = os.path.join(tokens[-2], tokens[-1])
        print('%s already exists in destination; skipping linking' % name)
        continue
      os.link(src, dest)

def main():
  parser = argparse.ArgumentParser()

  parser.add_argument(
      '--symbol-dirs',
      required=True,
      nargs='+',
      help='Space separated list of directories that contain the debug symbols.'
  )
  parser.add_argument(
      '--out-dir',
      required=True,
      action='store',
      dest='out_dir',
      help='Output directory where the executables will be placed.')
  parser.add_argument(
      '--target-arch', type=str, choices=['x64', 'arm64'], required=True)
  parser.add_argument(
      '--engine-version',
      required=True,
      help='Specifies the flutter engine SHA.')

  parser.add_argument('--upload', default=False, action='store_true')

  args = parser.parse_args()

  symbol_dirs = args.symbol_dirs
  for symbol_dir in symbol_dirs:
    assert os.path.exists(symbol_dir) and os.path.isdir(symbol_dir)

  out_dir = args.out_dir

  if os.path.exists(out_dir):
    print 'Directory: %s is not empty, deleting it.' % out_dir
    shutil.rmtree(out_dir)
  os.makedirs(out_dir)

  for symbol_dir in symbol_dirs:
    HardlinkContents(symbol_dir, out_dir)

  arch = args.target_arch
  cipd_def = WriteCIPDDefinition(arch, out_dir)
  ProcessCIPDPackage(args.upload, cipd_def, args.engine_version, out_dir, arch)
  return 0

if __name__ == '__main__':
  sys.exit(main())
<commit_msg>Revert "[tools][fuchsia] Do not tar debug symbol CIPD uploads (#17275)"<commit_after>#!/usr/bin/env python
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" Merges the debug symbols and uploads them to cipd.
"""

import argparse
import collections
import json
import os
import platform
import shutil
import subprocess
import sys
import tarfile

def IsLinux():
  return platform.system() == 'Linux'

def GetPackagingDir(out_dir):
  return os.path.abspath(os.path.join(out_dir, os.pardir))

def CreateCIPDDefinition(target_arch, out_dir):
  dir_name = os.path.basename(os.path.normpath(out_dir))
  return """
package: flutter/fuchsia-debug-symbols-%s
description: Flutter and Dart runner debug symbols for Fuchsia. Target architecture %s.
install_mode: copy
data:
  - dir: %s
""" % (target_arch, target_arch, dir_name)

# CIPD CLI needs the definition and data directory to be relative to each other.
def WriteCIPDDefinition(target_arch, out_dir):
  _packaging_dir = GetPackagingDir(out_dir)
  yaml_file = os.path.join(_packaging_dir, 'debug_symbols.cipd.yaml')
  with open(yaml_file, 'w') as f:
    cipd_def = CreateCIPDDefinition(target_arch, out_dir)
    f.write(cipd_def)
  return yaml_file

def ProcessCIPDPackage(upload, cipd_yaml, engine_version, out_dir, target_arch):
  _packaging_dir = GetPackagingDir(out_dir)
  if upload and IsLinux():
    command = [
        'cipd', 'create', '-pkg-def', cipd_yaml, '-ref', 'latest', '-tag',
        'git_revision:%s' % engine_version
    ]
  else:
    command = [
        'cipd', 'pkg-build', '-pkg-def', cipd_yaml, '-out',
        os.path.join(_packaging_dir,
                     'fuchsia-debug-symbols-%s.cipd' % target_arch)
    ]

  # Retry up to three times.  We've seen CIPD fail on verification in some
  # instances. Normally verification takes slightly more than 1 minute when
  # it succeeds.
  num_tries = 3
  for tries in range(num_tries):
    try:
      subprocess.check_call(command, cwd=_packaging_dir)
      break
    except subprocess.CalledProcessError:
      print('Failed %s times' % tries + 1)
      if tries == num_tries - 1:
        raise

def CreateTarFile(folder_path, base_dir):
  archive_name = os.path.basename(folder_path)
  tar_file_path = os.path.join(base_dir, archive_name + '.tar.bz2')
  with tarfile.open(tar_file_path, "w:bz2") as archive:
    for root, dirs, _ in os.walk(folder_path):
      for dir_name in dirs:
        dir_path = os.path.join(root, dir_name)
        archive.add(dir_path, arcname=dir_name)
  return tar_file_path

def main():
  parser = argparse.ArgumentParser()

  parser.add_argument(
      '--symbol-dirs',
      required=True,
      nargs='+',
      help='Space separated list of directories that contain the debug symbols.'
  )
  parser.add_argument(
      '--out-dir',
      required=True,
      action='store',
      dest='out_dir',
      help='Output directory where the executables will be placed.')
  parser.add_argument(
      '--target-arch', type=str, choices=['x64', 'arm64'], required=True)
  parser.add_argument(
      '--engine-version',
      required=True,
      help='Specifies the flutter engine SHA.')

  parser.add_argument('--upload', default=False, action='store_true')

  args = parser.parse_args()

  symbol_dirs = args.symbol_dirs
  for symbol_dir in symbol_dirs:
    assert os.path.exists(symbol_dir) and os.path.isdir(symbol_dir)

  arch = args.target_arch
  out_dir = os.path.join(args.out_dir,
                         'flutter-fuchsia-debug-symbols-%s' % arch)
  if os.path.exists(out_dir):
    print 'Directory: %s is not empty, deleting it.' % out_dir
    shutil.rmtree(out_dir)
  os.makedirs(out_dir)

  for symbol_dir in symbol_dirs:
    archive_path = CreateTarFile(symbol_dir, out_dir)
    print('Created archive: ' + archive_path)

  cipd_def = WriteCIPDDefinition(arch, out_dir)
  ProcessCIPDPackage(args.upload, cipd_def, args.engine_version, out_dir, arch)
  return 0

if __name__ == '__main__':
  sys.exit(main())
<|endoftext|>


<commit_before>########
# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import pytest
import time

from cosmo_tester.framework.examples.hello_world import centos_hello_world
from cosmo_tester.framework.test_hosts import TestHosts
from cosmo_tester.framework.util import prepare_and_get_test_tenant
from .ha_helper import HighAvailabilityHelper as ha_helper
from . import skip_community

# Skip all tests in this module if we're running community tests,
# using the pytestmark magic variable name
pytestmark = skip_community

@pytest.fixture(scope='function', params=[2, 3])
def hosts(
        request, cfy, ssh_key, module_tmpdir, attributes, logger):
    """Creates a HA cluster from an image in rackspace OpenStack."""
    logger.info('Creating HA cluster of %s managers', request.param)
    hosts = TestHosts(
            cfy, ssh_key, module_tmpdir, attributes, logger,
            number_of_instances=request.param)

    for manager in hosts.instances[1:]:
        manager.upload_plugins = False

    try:
        hosts.create()
        manager1 = hosts.instances[0]
        ha_helper.delete_active_profile()
        manager1.use()

        cfy.cluster.start(timeout=600,
                          cluster_host_ip=manager1.private_ip_address,
                          cluster_node_name=manager1.ip_address)

        for manager in hosts.instances[1:]:
            manager.use()
            cfy.cluster.join(manager1.ip_address,
                             timeout=600,
                             cluster_host_ip=manager.private_ip_address,
                             cluster_node_name=manager.ip_address)

        cfy.cluster.nodes.list()

        yield hosts

    finally:
        hosts.destroy()

@pytest.fixture(scope='function')
def ha_hello_worlds(cfy, hosts, attributes, ssh_key, tmpdir, logger):
    # Pick a manager to operate on, and trust the cluster to work with us
    manager = hosts.instances[0]

    hws = []
    for i in range(0, 2):
        tenant = prepare_and_get_test_tenant(
            'clusterhello{num}'.format(num=i),
            manager,
            cfy,
        )
        hw = centos_hello_world(
            cfy, manager, attributes, ssh_key, logger, tmpdir,
            tenant=tenant, suffix=str(i),
        )
        hws.append(hw)

    yield hws
    for hw in hws:
        if hw.cleanup_required:
            logger.info('Cleaning up hello world...')
            manager.use()
            hw.cleanup()

def test_data_replication(cfy, hosts, ha_hello_worlds, logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.verify_nodes_status(manager1, cfy, logger)
    _test_hellos(ha_hello_worlds, install=True)

    logger.info('Manager %s resources', manager1.ip_address)
    m1_blueprints_list = cfy.blueprints.list()
    m1_deployments_list = cfy.deployments.list()
    m1_plugins_list = cfy.plugins.list()

    for manager in hosts.instances[1:]:
        ha_helper.set_active(manager, cfy, logger)
        ha_helper.delete_active_profile()
        manager.use()
        ha_helper.verify_nodes_status(manager, cfy, logger)

        logger.info('Manager %s resources', manager.ip_address)
        assert m1_blueprints_list == cfy.blueprints.list()
        assert m1_deployments_list == cfy.deployments.list()
        assert m1_plugins_list == cfy.plugins.list()

    ha_helper.set_active(manager1, cfy, logger)
    ha_helper.delete_active_profile()
    manager1.use()

def test_set_active(cfy, hosts, logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.verify_nodes_status(manager1, cfy, logger)

    for manager in hosts.instances[1:]:
        ha_helper.set_active(manager, cfy, logger)
        ha_helper.delete_active_profile()
        manager.use()
        ha_helper.verify_nodes_status(manager, cfy, logger)

def test_delete_manager_node(cfy, hosts, ha_hello_worlds, logger):
    ha_helper.set_active(hosts.instances[1], cfy, logger)
    expected_master = hosts.instances[0]
    for manager in hosts.instances[1:]:
        logger.info('Deleting manager %s', manager.ip_address)
        manager.delete()
        ha_helper.wait_leader_election(
            [m for m in hosts.instances if not m.deleted], logger)

    logger.info('Expected leader %s', expected_master)
    ha_helper.verify_nodes_status(expected_master, cfy, logger)
    _test_hellos(ha_hello_worlds)

def test_failover(cfy, hosts, ha_hello_worlds, logger):
    """Test that the cluster fails over in case of a service failure

    - stop nginx on leader
    - check that a new leader is elected
    - stop mgmtworker on that new leader, and restart nginx on the former
    - check that the original leader was elected
    """
    expected_master = hosts.instances[-1]
    # stop nginx on all nodes except last - force choosing the last as the
    # leader (because only the last one has services running)
    for manager in hosts.instances[:-1]:
        logger.info('Simulating manager %s failure by stopping'
                    ' nginx service', manager.ip_address)
        with manager.ssh() as fabric:
            fabric.run('sudo systemctl stop nginx')
        # wait for checks to notice the service failure
        time.sleep(20)
        ha_helper.wait_leader_election(hosts.instances, logger)
        cfy.cluster.nodes.list()

    ha_helper.verify_nodes_status(expected_master, cfy, logger)

    new_expected_master = hosts.instances[0]
    # force going back to the original leader - start nginx on it, and
    # stop mgmtworker on the current leader (simulating failure)
    with new_expected_master.ssh() as fabric:
        logger.info('Starting nginx service on manager %s',
                    new_expected_master.ip_address)
        fabric.run('sudo systemctl start nginx')

    with expected_master.ssh() as fabric:
        logger.info('Simulating manager %s failure by stopping '
                    'cloudify-mgmtworker service',
                    expected_master.ip_address)
        fabric.run('sudo systemctl stop cloudify-mgmtworker')

    # wait for checks to notice the service failure
    time.sleep(20)
    ha_helper.wait_leader_election(hosts.instances, logger)
    cfy.cluster.nodes.list()

    ha_helper.verify_nodes_status(new_expected_master, cfy, logger)

    _test_hellos(ha_hello_worlds)

def test_remove_manager_from_cluster(cfy, hosts, ha_hello_worlds, logger):
    ha_helper.set_active(hosts.instances[1], cfy, logger)
    ha_helper.delete_active_profile()

    expected_master = hosts.instances[0]
    nodes_to_check = list(hosts.instances)
    for manager in hosts.instances[1:]:
        manager.use()
        logger.info('Removing the manager %s from HA cluster',
                    manager.ip_address)
        cfy.cluster.nodes.remove(manager.ip_address)
        nodes_to_check.remove(manager)
        ha_helper.wait_leader_election(nodes_to_check, logger)

    ha_helper.delete_active_profile()
    expected_master.use()

    ha_helper.verify_nodes_status(expected_master, cfy, logger)
    _test_hellos(ha_hello_worlds)

def test_uninstall_dep(cfy, hosts, ha_hello_worlds,
                       logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.verify_nodes_status(manager1, cfy, logger)
    _test_hellos(ha_hello_worlds, install=True)

    manager2 = hosts.instances[-1]
    ha_helper.set_active(manager2, cfy, logger)
    ha_helper.delete_active_profile()
    manager2.use()
    for hello_world in ha_hello_worlds:
        hello_world.uninstall()

def _test_hellos(hello_worlds, install=False):
    for hello_world in hello_worlds:
        hello_world.upload_blueprint()
        if install:
            hello_world.create_deployment()
            hello_world.install()
<commit_msg>HA tests: before set-active, wait for cluster to stabilize (#619)<commit_after>########
# Copyright (c) 2017 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import pytest
import time

from cosmo_tester.framework.examples.hello_world import centos_hello_world
from cosmo_tester.framework.test_hosts import TestHosts
from cosmo_tester.framework.util import prepare_and_get_test_tenant
from .ha_helper import HighAvailabilityHelper as ha_helper
from . import skip_community

# Skip all tests in this module if we're running community tests,
# using the pytestmark magic variable name
pytestmark = skip_community

@pytest.fixture(scope='function', params=[2, 3])
def hosts(
        request, cfy, ssh_key, module_tmpdir, attributes, logger):
    """Creates a HA cluster from an image in rackspace OpenStack."""
    logger.info('Creating HA cluster of %s managers', request.param)
    hosts = TestHosts(
            cfy, ssh_key, module_tmpdir, attributes, logger,
            number_of_instances=request.param)

    for manager in hosts.instances[1:]:
        manager.upload_plugins = False

    try:
        hosts.create()
        manager1 = hosts.instances[0]
        ha_helper.delete_active_profile()
        manager1.use()

        cfy.cluster.start(timeout=600,
                          cluster_host_ip=manager1.private_ip_address,
                          cluster_node_name=manager1.ip_address)

        for manager in hosts.instances[1:]:
            manager.use()
            cfy.cluster.join(manager1.ip_address,
                             timeout=600,
                             cluster_host_ip=manager.private_ip_address,
                             cluster_node_name=manager.ip_address)

        cfy.cluster.nodes.list()

        yield hosts

    finally:
        hosts.destroy()

@pytest.fixture(scope='function')
def ha_hello_worlds(cfy, hosts, attributes, ssh_key, tmpdir, logger):
    # Pick a manager to operate on, and trust the cluster to work with us
    manager = hosts.instances[0]

    hws = []
    for i in range(0, 2):
        tenant = prepare_and_get_test_tenant(
            'clusterhello{num}'.format(num=i),
            manager,
            cfy,
        )
        hw = centos_hello_world(
            cfy, manager, attributes, ssh_key, logger, tmpdir,
            tenant=tenant, suffix=str(i),
        )
        hws.append(hw)

    yield hws
    for hw in hws:
        if hw.cleanup_required:
            logger.info('Cleaning up hello world...')
            manager.use()
            hw.cleanup()

def test_data_replication(cfy, hosts, ha_hello_worlds, logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.verify_nodes_status(manager1, cfy, logger)
    _test_hellos(ha_hello_worlds, install=True)

    logger.info('Manager %s resources', manager1.ip_address)
    m1_blueprints_list = cfy.blueprints.list()
    m1_deployments_list = cfy.deployments.list()
    m1_plugins_list = cfy.plugins.list()

    for manager in hosts.instances[1:]:
        ha_helper.set_active(manager, cfy, logger)
        ha_helper.delete_active_profile()
        manager.use()
        ha_helper.verify_nodes_status(manager, cfy, logger)

        logger.info('Manager %s resources', manager.ip_address)
        assert m1_blueprints_list == cfy.blueprints.list()
        assert m1_deployments_list == cfy.deployments.list()
        assert m1_plugins_list == cfy.plugins.list()

    ha_helper.set_active(manager1, cfy, logger)
    ha_helper.delete_active_profile()
    manager1.use()

def test_set_active(cfy, hosts, logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.wait_nodes_online(hosts.instances, logger)
    ha_helper.verify_nodes_status(manager1, cfy, logger)

    for manager in hosts.instances[1:]:
        ha_helper.set_active(manager, cfy, logger)
        ha_helper.delete_active_profile()
        manager.use()
        ha_helper.verify_nodes_status(manager, cfy, logger)

def test_delete_manager_node(cfy, hosts, ha_hello_worlds, logger):
    ha_helper.set_active(hosts.instances[1], cfy, logger)
    expected_master = hosts.instances[0]
    for manager in hosts.instances[1:]:
        logger.info('Deleting manager %s', manager.ip_address)
        manager.delete()
        ha_helper.wait_leader_election(
            [m for m in hosts.instances if not m.deleted], logger)

    logger.info('Expected leader %s', expected_master)
    ha_helper.verify_nodes_status(expected_master, cfy, logger)
    _test_hellos(ha_hello_worlds)

def test_failover(cfy, hosts, ha_hello_worlds, logger):
    """Test that the cluster fails over in case of a service failure

    - stop nginx on leader
    - check that a new leader is elected
    - stop mgmtworker on that new leader, and restart nginx on the former
    - check that the original leader was elected
    """
    expected_master = hosts.instances[-1]
    # stop nginx on all nodes except last - force choosing the last as the
    # leader (because only the last one has services running)
    for manager in hosts.instances[:-1]:
        logger.info('Simulating manager %s failure by stopping'
                    ' nginx service', manager.ip_address)
        with manager.ssh() as fabric:
            fabric.run('sudo systemctl stop nginx')
        # wait for checks to notice the service failure
        time.sleep(20)
        ha_helper.wait_leader_election(hosts.instances, logger)
        cfy.cluster.nodes.list()

    ha_helper.verify_nodes_status(expected_master, cfy, logger)

    new_expected_master = hosts.instances[0]
    # force going back to the original leader - start nginx on it, and
    # stop mgmtworker on the current leader (simulating failure)
    with new_expected_master.ssh() as fabric:
        logger.info('Starting nginx service on manager %s',
                    new_expected_master.ip_address)
        fabric.run('sudo systemctl start nginx')

    with expected_master.ssh() as fabric:
        logger.info('Simulating manager %s failure by stopping '
                    'cloudify-mgmtworker service',
                    expected_master.ip_address)
        fabric.run('sudo systemctl stop cloudify-mgmtworker')

    # wait for checks to notice the service failure
    time.sleep(20)
    ha_helper.wait_leader_election(hosts.instances, logger)
    cfy.cluster.nodes.list()

    ha_helper.verify_nodes_status(new_expected_master, cfy, logger)

    _test_hellos(ha_hello_worlds)

def test_remove_manager_from_cluster(cfy, hosts, ha_hello_worlds, logger):
    ha_helper.set_active(hosts.instances[1], cfy, logger)
    ha_helper.delete_active_profile()

    expected_master = hosts.instances[0]
    nodes_to_check = list(hosts.instances)
    for manager in hosts.instances[1:]:
        manager.use()
        logger.info('Removing the manager %s from HA cluster',
                    manager.ip_address)
        cfy.cluster.nodes.remove(manager.ip_address)
        nodes_to_check.remove(manager)
        ha_helper.wait_leader_election(nodes_to_check, logger)

    ha_helper.delete_active_profile()
    expected_master.use()

    ha_helper.verify_nodes_status(expected_master, cfy, logger)
    _test_hellos(ha_hello_worlds)

def test_uninstall_dep(cfy, hosts, ha_hello_worlds,
                       logger):
    manager1 = hosts.instances[0]
    ha_helper.delete_active_profile()
    manager1.use()
    ha_helper.verify_nodes_status(manager1, cfy, logger)
    _test_hellos(ha_hello_worlds, install=True)

    manager2 = hosts.instances[-1]
    ha_helper.set_active(manager2, cfy, logger)
    ha_helper.delete_active_profile()
    manager2.use()
    for hello_world in ha_hello_worlds:
        hello_world.uninstall()

def _test_hellos(hello_worlds, install=False):
    for hello_world in hello_worlds:
        hello_world.upload_blueprint()
        if install:
            hello_world.create_deployment()
            hello_world.install()
<|endoftext|>


<commit_before>from django import template
from django.core.cache import cache
from django.db.models import Sum

from packages.models import Package, Release, ReleaseFile, ChangeLog

register = template.Library()

@register.assignment_tag
def package_download_count(package_name=None):
    if package_name is None:
        cached = cache.get("crate:stats:download_count")

        if cached:
            return cached

        count = ReleaseFile.objects.all().aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
        cache.set("crate:stats:download_count", count, 60 * 60)
        return count
    else:
        cached = cache.get("crate:stats:download_count:%s" % package_name)

        if cached:
            return cached

        count = ReleaseFile.objects.filter(
                    release__package__name=package_name
                ).aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
        cache.set("crate:stats:download_count:%s" % package_name, count, 60 * 60 * 24)
        return count

@register.assignment_tag
def package_count():
    cached = cache.get("crate:stats:package_count")

    if cached:
        return cached

    count = Package.objects.all().count()
    cache.set("crate:stats:package_count", count, 60 * 60)
    return count

@register.assignment_tag
def get_oldest_package():
    cached = cache.get("crate:stats:oldest_package")

    if cached:
        return cached

    pkgs = Package.objects.all().order_by("created")[:1]

    if pkgs:
        cache.set("crate:stats:oldest_package", pkgs[0], 60 * 60 * 24 * 7)
        return pkgs[0]
    else:
        return None

@register.assignment_tag
def new_packages(num):
    return [
        x for
        x in ChangeLog.objects.filter(type=ChangeLog.TYPES.new).select_related("package", "release").prefetch_related("package__releases").order_by("-created")[:num * 3]
        if len(x.package.releases.all())
    ][:num]

@register.assignment_tag
def updated_packages(num):
    return ChangeLog.objects.filter(type=ChangeLog.TYPES.updated).select_related("package", "release", "release__package").order_by("-created")[:num]

@register.assignment_tag
def featured_packages(num):
    return Package.objects.filter(featured=True).order_by("?")[:num]

@register.assignment_tag
def random_packages(num):
    return Package.objects.exclude(releases=None).order_by("?")[:num]

@register.assignment_tag
def package_versions(package_name, num=None):
    qs = Release.objects.filter(package__name=package_name).select_related("package").order_by("-order")
    if num is not None:
        qs = qs[:num]
    return qs

@register.assignment_tag
def package_version_count(package_name):
    return Release.objects.filter(package__name=package_name).count()
<commit_msg>filter out deleted releases from versions<commit_after>from django import template
from django.core.cache import cache
from django.db.models import Sum

from packages.models import Package, Release, ReleaseFile, ChangeLog

register = template.Library()

@register.assignment_tag
def package_download_count(package_name=None):
    if package_name is None:
        cached = cache.get("crate:stats:download_count")

        if cached:
            return cached

        count = ReleaseFile.objects.all().aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
        cache.set("crate:stats:download_count", count, 60 * 60)
        return count
    else:
        cached = cache.get("crate:stats:download_count:%s" % package_name)

        if cached:
            return cached

        count = ReleaseFile.objects.filter(
                    release__package__name=package_name
                ).aggregate(total_downloads=Sum("downloads")).get("total_downloads", 0)
        cache.set("crate:stats:download_count:%s" % package_name, count, 60 * 60 * 24)
        return count

@register.assignment_tag
def package_count():
    cached = cache.get("crate:stats:package_count")

    if cached:
        return cached

    count = Package.objects.all().count()
    cache.set("crate:stats:package_count", count, 60 * 60)
    return count

@register.assignment_tag
def get_oldest_package():
    cached = cache.get("crate:stats:oldest_package")

    if cached:
        return cached

    pkgs = Package.objects.all().order_by("created")[:1]

    if pkgs:
        cache.set("crate:stats:oldest_package", pkgs[0], 60 * 60 * 24 * 7)
        return pkgs[0]
    else:
        return None

@register.assignment_tag
def new_packages(num):
    return [
        x for
        x in ChangeLog.objects.filter(type=ChangeLog.TYPES.new).select_related("package", "release").prefetch_related("package__releases").order_by("-created")[:num * 3]
        if len(x.package.releases.all())
    ][:num]

@register.assignment_tag
def updated_packages(num):
    return ChangeLog.objects.filter(type=ChangeLog.TYPES.updated).select_related("package", "release", "release__package").order_by("-created")[:num]

@register.assignment_tag
def featured_packages(num):
    return Package.objects.filter(featured=True).order_by("?")[:num]

@register.assignment_tag
def random_packages(num):
    return Package.objects.exclude(releases=None).order_by("?")[:num]

@register.assignment_tag
def package_versions(package_name, num=None):
    qs = Release.objects.filter(package__name=package_name, deleted=False).select_related("package").order_by("-order")
    if num is not None:
        qs = qs[:num]
    return qs

@register.assignment_tag
def package_version_count(package_name):
    return Release.objects.filter(package__name=package_name).count()
<|endoftext|>


<commit_before># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Used for compilation by the different implementations of build_defs.bzl.
"""

load(":common/module_mappings.bzl", "module_mappings_aspect")
load(":common/json_marshal.bzl", "json_marshal")

BASE_ATTRIBUTES = dict()

DEPS_ASPECTS = [
    module_mappings_aspect,
]

# Attributes shared by any typescript-compatible rule (ts_library, ng_module)
COMMON_ATTRIBUTES = dict(BASE_ATTRIBUTES, **{
    "deps": attr.label_list(aspects = DEPS_ASPECTS),
    "data": attr.label_list(
        default = [],
        allow_files = True,
        cfg = "data",
    ),
    # TODO(evanm): make this the default and remove the option.
    "runtime": attr.string(default="browser"),
    # Used to determine module mappings
    "module_name": attr.string(),
    "module_root": attr.string(),
    # TODO(radokirov): remove this attr when clutz is stable enough to consume
    # any closure JS code.
    "runtime_deps": attr.label_list(
        default = [],
        providers = ["js"],
    ),
    # Override _additional_d_ts to specify google3 stdlibs
    "_additional_d_ts": attr.label_list(
        allow_files = True,
    ),
    # Whether to generate externs.js from any "declare" statement.
    "generate_externs": attr.bool(default = True),
})

COMMON_OUTPUTS = {
    # Allow the tsconfig.json to be generated without running compile actions.
    "tsconfig": "%{name}_tsconfig.json"
}

# TODO(plf): Enforce this at analysis time.
def assert_js_or_typescript_deps(ctx):
  for dep in ctx.attr.deps:
    if not hasattr(dep, "typescript") and not hasattr(dep, "js"):
      fail(
          ("%s is neither a TypeScript nor a JS producing rule." % dep.label) +
          "\nDependencies must be ts_library, ts_declaration, or " +
          # TODO(plf): Leaving this here for now, but this message does not
          # make sense in opensource.
          "JavaScript library rules (js_library, pinto_library, etc, but " +
          "also proto_library and some others).\n")

def _collect_transitive_dts(ctx):
  all_deps_declarations = depset()
  type_blacklisted_declarations = depset()
  for extra in ctx.files._additional_d_ts:
    all_deps_declarations += depset([extra])
  for dep in ctx.attr.deps:
    if hasattr(dep, "typescript"):
      all_deps_declarations += dep.typescript.transitive_declarations
      type_blacklisted_declarations += (
          dep.typescript.type_blacklisted_declarations)
  return struct(
      transitive_declarations=list(all_deps_declarations),
      type_blacklisted_declarations=list(type_blacklisted_declarations)
  )

def _outputs(ctx, label):
  """Returns closure js, devmode js, and .d.ts output files.

  Args:
    ctx: ctx.
    label: Label. package label.
  Returns:
    A struct of file lists for different output types.
  """
  workspace_segments = label.workspace_root.split("/") if label.workspace_root else []
  package_segments = label.package.split("/") if label.package else []
  trim = len(workspace_segments) + len(package_segments)
  closure_js_files = []
  devmode_js_files = []
  declaration_files = []
  for input_file in ctx.files.srcs:
    if (input_file.short_path.endswith(".d.ts")):
      continue
    basename = "/".join(input_file.short_path.split("/")[trim:])
    dot = basename.rfind(".")
    basename = basename[:dot]
    closure_js_files += [ctx.new_file(basename + ".closure.js")]
    devmode_js_files += [ctx.new_file(basename + ".js")]
    declaration_files += [ctx.new_file(basename + ".d.ts")]
  return struct(
    closure_js = closure_js_files,
    devmode_js = devmode_js_files,
    declarations = declaration_files,
  )

def compile_ts(ctx,
               is_library,
               extra_dts_files=[],
               compile_action=None,
               devmode_compile_action=None,
               jsx_factory=None,
               tsc_wrapped_tsconfig=None,
               outputs=_outputs):
  """Creates actions to compile TypeScript code.

  This rule is shared between ts_library and ts_declaration.

  Args:
    ctx: ctx.
    is_library: boolean. False if only compiling .dts files.
    extra_dts_files: list. Additional dts files to pass for compilation,
      not included in the transitive closure of declarations.
    compile_action: function. Creates the compilation action.
    devmode_compile_action: function. Creates the compilation action
      for devmode.
    jsx_factory: optional string. Enables overriding jsx pragma.
    tsc_wrapped_tsconfig: function that produces a tsconfig object.
    outputs: function from a ctx to the expected compilation outputs.
  Returns:
    struct that will be returned by the rule implementation.
  """
  assert_js_or_typescript_deps(ctx)

  ### Collect srcs and outputs.
  srcs = ctx.files.srcs
  src_declarations = []  # d.ts found in inputs.
  tsickle_externs = []  # externs.js generated by tsickle, if any.
  has_sources = False

  # Validate the user inputs.
  for src in ctx.attr.srcs:
    if src.label.package != ctx.label.package:
      # Sources can be in sub-folders, but not in sub-packages.
      fail("Sources must be in the same package as the ts_library rule, " +
           "but %s is not in %s" % (src.label, ctx.label.package), "srcs")

    for f in src.files:
      has_sources = True
      if not is_library and not f.path.endswith(".d.ts"):
          fail("srcs must contain only type declarations (.d.ts files), " +
               "but %s contains %s" % (src.label, f.short_path), "srcs")
      if f.path.endswith(".d.ts"):
        src_declarations += [f]
        continue

  outs = outputs(ctx, ctx.label)
  transpiled_closure_js = outs.closure_js
  transpiled_devmode_js = outs.devmode_js
  gen_declarations = outs.declarations

  if has_sources and ctx.attr.runtime != "nodejs":
    # Note: setting this variable controls whether tsickle is run at all.
    tsickle_externs = [ctx.new_file(ctx.label.name + ".externs.js")]

  transitive_dts = _collect_transitive_dts(ctx)
  input_declarations = transitive_dts.transitive_declarations + src_declarations
  type_blacklisted_declarations = transitive_dts.type_blacklisted_declarations
  if not is_library and not ctx.attr.generate_externs:
    type_blacklisted_declarations += ctx.files.srcs

  # The list of output files. These are the files that are always built
  # (including e.g. if you "blaze build :the_target" directly).
  files = depset()

  # A manifest listing the order of this rule's *.ts files (non-transitive)
  # Only generated if the rule has any sources.
  devmode_manifest = None

  # Enable to produce a performance trace when compiling TypeScript to JS.
  # The trace file location will be printed as a build result and can be read
  # in Chrome's chrome://tracing/ UI.
  perf_trace = False

  compilation_inputs = input_declarations + extra_dts_files + srcs
  tsickle_externs_path = tsickle_externs[0] if tsickle_externs else None

  # Calculate allowed dependencies for strict deps enforcement.
  allowed_deps = srcs[:]  # A target's sources may depend on each other.
  for dep in ctx.attr.deps:
    if hasattr(dep, "typescript"):
      allowed_deps += dep.typescript.declarations
  allowed_deps += extra_dts_files

  tsconfig_es6 = tsc_wrapped_tsconfig(
      ctx,
      compilation_inputs,
      srcs,
      jsx_factory=jsx_factory,
      tsickle_externs=tsickle_externs_path,
      type_blacklisted_declarations=type_blacklisted_declarations,
      allowed_deps=allowed_deps)
  # Do not produce declarations in ES6 mode, tsickle cannot produce correct
  # .d.ts (or even errors) from the altered Closure-style JS emit.
  tsconfig_es6["compilerOptions"]["declaration"] = False
  tsconfig_es6["compilerOptions"].pop("declarationDir")
  outputs = transpiled_closure_js + tsickle_externs
  if perf_trace:
    perf_trace_file = ctx.new_file(ctx.label.name + ".es6.trace")
    tsconfig_es6["bazelOptions"]["perfTracePath"] = perf_trace_file.path
    outputs.append(perf_trace_file)
    files += [perf_trace_file]
  ctx.file_action(output=ctx.outputs.tsconfig,
                  content=json_marshal(tsconfig_es6))

  # Parameters of this compiler invocation in case we need to replay this with different
  # settings for I18N.
  replay_params = None

  if has_sources:
    inputs = compilation_inputs + [ctx.outputs.tsconfig]
    replay_params = compile_action(ctx, inputs, outputs, ctx.outputs.tsconfig.path)

    devmode_manifest = ctx.new_file(ctx.label.name + ".es5.MF")
    tsconfig_json_es5 = ctx.new_file(ctx.label.name + "_es5_tsconfig.json")
    outputs = (
        transpiled_devmode_js + gen_declarations + [devmode_manifest])
    tsconfig_es5 = tsc_wrapped_tsconfig(ctx,
                                        compilation_inputs,
                                        srcs,
                                        jsx_factory=jsx_factory,
                                        devmode_manifest=devmode_manifest.path,
                                        allowed_deps=allowed_deps)
    if perf_trace:
      perf_trace_file = ctx.new_file(ctx.label.name + ".es5.trace")
      tsconfig_es5["bazelOptions"]["perfTracePath"] = perf_trace_file.path
      outputs.append(perf_trace_file)
      files += [perf_trace_file]
    ctx.file_action(output=tsconfig_json_es5, content=json_marshal(
        tsconfig_es5))
    inputs = compilation_inputs + [tsconfig_json_es5]
    devmode_compile_action(ctx, inputs, outputs, tsconfig_json_es5.path)

  # TODO(martinprobst): Merge the generated .d.ts files, and enforce strict
  # deps (do not re-export transitive types from the transitive closure).
  transitive_decls = input_declarations + gen_declarations

  if is_library:
    es6_sources = depset(transpiled_closure_js + tsickle_externs)
    es5_sources = depset(transpiled_devmode_js)
  else:
    es6_sources = depset(tsickle_externs)
    es5_sources = depset(tsickle_externs)
    devmode_manifest = None

  # Downstream rules see the .d.ts files produced or declared by this rule.
  declarations = gen_declarations + src_declarations
  if not srcs:
    # Re-export sources from deps.
    # TODO(b/30018387): introduce an "exports" attribute.
    for dep in ctx.attr.deps:
      if hasattr(dep, "typescript"):
        declarations += dep.typescript.declarations
  files += declarations

  # If this is a ts_declaration, add tsickle_externs to the outputs list to
  # force compilation of d.ts files.  (tsickle externs are produced by running a
  # compilation over the d.ts file and extracting type information.)
  if not is_library:
    files += depset(tsickle_externs)

  return {
      "files": files,
      "runfiles": ctx.runfiles(
          # Note: don't include files=... here, or they will *always* be built
          # by any dependent rule, regardless of whether it needs them.
          # But these attributes are needed to pass along any input runfiles:
          collect_default=True,
          collect_data=True,
      ),
      # TODO(martinprobst): Prune transitive deps, only re-export what's needed.
      "typescript": {
          "declarations": declarations,
          "transitive_declarations": transitive_decls,
          "es6_sources": es6_sources,
          "es5_sources": es5_sources,
          "devmode_manifest": devmode_manifest,
          "type_blacklisted_declarations": type_blacklisted_declarations,
          "tsickle_externs": tsickle_externs,
          "replay_params": replay_params,
      },
      # Expose the tags so that a Skylark aspect can access them.
      "tags": ctx.attr.tags,
      "instrumented_files": {
          "extensions": ["ts"],
          "source_attributes": ["srcs"],
          "dependency_attributes": ["deps", "runtime_deps"],
      },
  }

# Converts a dict to a struct, recursing into a single level of nested dicts.
# This allows users of compile_ts to modify or augment the returned dict before
# converting it to an immutable struct.
def ts_providers_dict_to_struct(d):
  for key, value in d.items():
    if type(value) == type({}):
      d[key] = struct(**value)
  return struct(**d)
<commit_msg>Do not flatten depsets in typescript rules.<commit_after># Copyright 2017 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Used for compilation by the different implementations of build_defs.bzl.
"""

load(":common/module_mappings.bzl", "module_mappings_aspect")
load(":common/json_marshal.bzl", "json_marshal")

BASE_ATTRIBUTES = dict()

DEPS_ASPECTS = [
    module_mappings_aspect,
]

# Attributes shared by any typescript-compatible rule (ts_library, ng_module)
COMMON_ATTRIBUTES = dict(BASE_ATTRIBUTES, **{
    "deps": attr.label_list(aspects = DEPS_ASPECTS),
    "data": attr.label_list(
        default = [],
        allow_files = True,
        cfg = "data",
    ),
    # TODO(evanm): make this the default and remove the option.
    "runtime": attr.string(default="browser"),
    # Used to determine module mappings
    "module_name": attr.string(),
    "module_root": attr.string(),
    # TODO(radokirov): remove this attr when clutz is stable enough to consume
    # any closure JS code.
    "runtime_deps": attr.label_list(
        default = [],
        providers = ["js"],
    ),
    # Override _additional_d_ts to specify google3 stdlibs
    "_additional_d_ts": attr.label_list(
        allow_files = True,
    ),
    # Whether to generate externs.js from any "declare" statement.
    "generate_externs": attr.bool(default = True),
})

COMMON_OUTPUTS = {
    # Allow the tsconfig.json to be generated without running compile actions.
    "tsconfig": "%{name}_tsconfig.json"
}

# TODO(plf): Enforce this at analysis time.
def assert_js_or_typescript_deps(ctx):
  for dep in ctx.attr.deps:
    if not hasattr(dep, "typescript") and not hasattr(dep, "js"):
      fail(
          ("%s is neither a TypeScript nor a JS producing rule." % dep.label) +
          "\nDependencies must be ts_library, ts_declaration, or " +
          # TODO(plf): Leaving this here for now, but this message does not
          # make sense in opensource.
          "JavaScript library rules (js_library, pinto_library, etc, but " +
          "also proto_library and some others).\n")

def _collect_transitive_dts(ctx):
  all_deps_declarations = depset()
  type_blacklisted_declarations = depset()
  for extra in ctx.files._additional_d_ts:
    all_deps_declarations += depset([extra])
  for dep in ctx.attr.deps:
    if hasattr(dep, "typescript"):
      all_deps_declarations += dep.typescript.transitive_declarations
      type_blacklisted_declarations += dep.typescript.type_blacklisted_declarations
  return struct(
      transitive_declarations=all_deps_declarations,
      type_blacklisted_declarations=type_blacklisted_declarations
  )

def _outputs(ctx, label):
  """Returns closure js, devmode js, and .d.ts output files.

  Args:
    ctx: ctx.
    label: Label. package label.
  Returns:
    A struct of file lists for different output types.
  """
  workspace_segments = label.workspace_root.split("/") if label.workspace_root else []
  package_segments = label.package.split("/") if label.package else []
  trim = len(workspace_segments) + len(package_segments)
  closure_js_files = []
  devmode_js_files = []
  declaration_files = []
  for input_file in ctx.files.srcs:
    if (input_file.short_path.endswith(".d.ts")):
      continue
    basename = "/".join(input_file.short_path.split("/")[trim:])
    dot = basename.rfind(".")
    basename = basename[:dot]
    closure_js_files += [ctx.new_file(basename + ".closure.js")]
    devmode_js_files += [ctx.new_file(basename + ".js")]
    declaration_files += [ctx.new_file(basename + ".d.ts")]
  return struct(
    closure_js = closure_js_files,
    devmode_js = devmode_js_files,
    declarations = declaration_files,
  )

def compile_ts(ctx,
               is_library,
               extra_dts_files=[],
               compile_action=None,
               devmode_compile_action=None,
               jsx_factory=None,
               tsc_wrapped_tsconfig=None,
               outputs=_outputs):
  """Creates actions to compile TypeScript code.

  This rule is shared between ts_library and ts_declaration.

  Args:
    ctx: ctx.
    is_library: boolean. False if only compiling .dts files.
    extra_dts_files: list. Additional dts files to pass for compilation,
      not included in the transitive closure of declarations.
    compile_action: function. Creates the compilation action.
    devmode_compile_action: function. Creates the compilation action
      for devmode.
    jsx_factory: optional string. Enables overriding jsx pragma.
    tsc_wrapped_tsconfig: function that produces a tsconfig object.
    outputs: function from a ctx to the expected compilation outputs.
  Returns:
    struct that will be returned by the rule implementation.
  """
  assert_js_or_typescript_deps(ctx)

  ### Collect srcs and outputs.
  srcs = ctx.files.srcs
  src_declarations = []  # d.ts found in inputs.
  tsickle_externs = []  # externs.js generated by tsickle, if any.
  has_sources = False

  # Validate the user inputs.
  for src in ctx.attr.srcs:
    if src.label.package != ctx.label.package:
      # Sources can be in sub-folders, but not in sub-packages.
      fail("Sources must be in the same package as the ts_library rule, " +
           "but %s is not in %s" % (src.label, ctx.label.package), "srcs")

    for f in src.files:
      has_sources = True
      if not is_library and not f.path.endswith(".d.ts"):
          fail("srcs must contain only type declarations (.d.ts files), " +
               "but %s contains %s" % (src.label, f.short_path), "srcs")
      if f.path.endswith(".d.ts"):
        src_declarations += [f]
        continue

  outs = outputs(ctx, ctx.label)
  transpiled_closure_js = outs.closure_js
  transpiled_devmode_js = outs.devmode_js
  gen_declarations = outs.declarations

  if has_sources and ctx.attr.runtime != "nodejs":
    # Note: setting this variable controls whether tsickle is run at all.
    tsickle_externs = [ctx.new_file(ctx.label.name + ".externs.js")]

  transitive_dts = _collect_transitive_dts(ctx)
  input_declarations = transitive_dts.transitive_declarations + src_declarations
  type_blacklisted_declarations = transitive_dts.type_blacklisted_declarations
  if not is_library and not ctx.attr.generate_externs:
    type_blacklisted_declarations += ctx.files.srcs

  # The list of output files. These are the files that are always built
  # (including e.g. if you "blaze build :the_target" directly).
  files = depset()

  # A manifest listing the order of this rule's *.ts files (non-transitive)
  # Only generated if the rule has any sources.
  devmode_manifest = None

  # Enable to produce a performance trace when compiling TypeScript to JS.
  # The trace file location will be printed as a build result and can be read
  # in Chrome's chrome://tracing/ UI.
  perf_trace = False

  compilation_inputs = input_declarations + extra_dts_files + srcs
  tsickle_externs_path = tsickle_externs[0] if tsickle_externs else None

  # Calculate allowed dependencies for strict deps enforcement.
  allowed_deps = depset()
  allowed_deps += srcs[:]  # A target's sources may depend on each other.
  for dep in ctx.attr.deps:
    if hasattr(dep, "typescript"):
      allowed_deps += dep.typescript.declarations
  allowed_deps += extra_dts_files

  tsconfig_es6 = tsc_wrapped_tsconfig(
      ctx,
      compilation_inputs,
      srcs,
      jsx_factory=jsx_factory,
      tsickle_externs=tsickle_externs_path,
      type_blacklisted_declarations=type_blacklisted_declarations,
      allowed_deps=allowed_deps)
  # Do not produce declarations in ES6 mode, tsickle cannot produce correct
  # .d.ts (or even errors) from the altered Closure-style JS emit.
  tsconfig_es6["compilerOptions"]["declaration"] = False
  tsconfig_es6["compilerOptions"].pop("declarationDir")
  outputs = transpiled_closure_js + tsickle_externs
  if perf_trace:
    perf_trace_file = ctx.new_file(ctx.label.name + ".es6.trace")
    tsconfig_es6["bazelOptions"]["perfTracePath"] = perf_trace_file.path
    outputs.append(perf_trace_file)
    files += [perf_trace_file]
  ctx.file_action(output=ctx.outputs.tsconfig,
                  content=json_marshal(tsconfig_es6))

  # Parameters of this compiler invocation in case we need to replay this with different
  # settings for I18N.
  replay_params = None

  if has_sources:
    inputs = compilation_inputs + [ctx.outputs.tsconfig]
    replay_params = compile_action(ctx, inputs, outputs, ctx.outputs.tsconfig.path)

    devmode_manifest = ctx.new_file(ctx.label.name + ".es5.MF")
    tsconfig_json_es5 = ctx.new_file(ctx.label.name + "_es5_tsconfig.json")
    outputs = (
        transpiled_devmode_js + gen_declarations + [devmode_manifest])
    tsconfig_es5 = tsc_wrapped_tsconfig(ctx,
                                        compilation_inputs,
                                        srcs,
                                        jsx_factory=jsx_factory,
                                        devmode_manifest=devmode_manifest.path,
                                        allowed_deps=allowed_deps)
    if perf_trace:
      perf_trace_file = ctx.new_file(ctx.label.name + ".es5.trace")
      tsconfig_es5["bazelOptions"]["perfTracePath"] = perf_trace_file.path
      outputs.append(perf_trace_file)
      files += [perf_trace_file]
    ctx.file_action(output=tsconfig_json_es5, content=json_marshal(
        tsconfig_es5))
    inputs = compilation_inputs + [tsconfig_json_es5]
    devmode_compile_action(ctx, inputs, outputs, tsconfig_json_es5.path)

  # TODO(martinprobst): Merge the generated .d.ts files, and enforce strict
  # deps (do not re-export transitive types from the transitive closure).
  transitive_decls = input_declarations + gen_declarations

  if is_library:
    es6_sources = depset(transpiled_closure_js + tsickle_externs)
    es5_sources = depset(transpiled_devmode_js)
  else:
    es6_sources = depset(tsickle_externs)
    es5_sources = depset(tsickle_externs)
    devmode_manifest = None

  # Downstream rules see the .d.ts files produced or declared by this rule.
  declarations = depset()
  declarations += gen_declarations
  declarations += src_declarations
  if not srcs:
    # Re-export sources from deps.
    # TODO(b/30018387): introduce an "exports" attribute.
    for dep in ctx.attr.deps:
      if hasattr(dep, "typescript"):
        declarations += dep.typescript.declarations
  files += declarations

  # If this is a ts_declaration, add tsickle_externs to the outputs list to
  # force compilation of d.ts files.  (tsickle externs are produced by running a
  # compilation over the d.ts file and extracting type information.)
  if not is_library:
    files += depset(tsickle_externs)

  return {
      "files": files,
      "runfiles": ctx.runfiles(
          # Note: don't include files=... here, or they will *always* be built
          # by any dependent rule, regardless of whether it needs them.
          # But these attributes are needed to pass along any input runfiles:
          collect_default=True,
          collect_data=True,
      ),
      # TODO(martinprobst): Prune transitive deps, only re-export what's needed.
      "typescript": {
          "declarations": declarations,
          "transitive_declarations": transitive_decls,
          "es6_sources": es6_sources,
          "es5_sources": es5_sources,
          "devmode_manifest": devmode_manifest,
          "type_blacklisted_declarations": type_blacklisted_declarations,
          "tsickle_externs": tsickle_externs,
          "replay_params": replay_params,
      },
      # Expose the tags so that a Skylark aspect can access them.
      "tags": ctx.attr.tags,
      "instrumented_files": {
          "extensions": ["ts"],
          "source_attributes": ["srcs"],
          "dependency_attributes": ["deps", "runtime_deps"],
      },
  }

# Converts a dict to a struct, recursing into a single level of nested dicts.
# This allows users of compile_ts to modify or augment the returned dict before
# converting it to an immutable struct.
def ts_providers_dict_to_struct(d):
  for key, value in d.items():
    if type(value) == type({}):
      d[key] = struct(**value)
  return struct(**d)
<|endoftext|>


<commit_before>from plugging_it_in.models import TestCase

from tests.BaseTestWithDB import BaseTestWithDB
from tests.topics.TopicsTestDataGenerator import TopicsTestDataGenerator

class TestCaseModelTest(BaseTestWithDB):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.test_data = TopicsTestDataGenerator()

    def create_testcase(self):
        topic = self.test_data.create_topic(1)
        difficulty = self.test_data.create_difficulty_level(1)
        challenge = self.test_data.create_programming_challenge(topic, 1, difficulty)

        self.test_data.create_programming_challenge_test_case(1, challenge)
        self.test_case = TestCase.objects.get(id=1)

    def test_testcase_verbose_model_name(self):
        self.create_testcase()
        verbose_name = self.test_case._meta.verbose_name
        self.assertEquals(verbose_name, "Test Case")
<commit_msg>Fix models unit test for plugging it in<commit_after>from tests.BaseTestWithDB import BaseTestWithDB
from tests.topics.TopicsTestDataGenerator import TopicsTestDataGenerator

class TestCaseModelTest(BaseTestWithDB):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.test_data = TopicsTestDataGenerator()

    def create_testcase(self):
        topic = self.test_data.create_topic(1)
        difficulty = self.test_data.create_difficulty_level(1)
        challenge = self.test_data.create_programming_challenge(topic, 1, difficulty)

        self.test_case = self.test_data.create_programming_challenge_test_case(1, challenge)

    def test_testcase_verbose_model_name(self):
        self.create_testcase()
        verbose_name = self.test_case._meta.verbose_name
        self.assertEquals(verbose_name, "Test Case")
<|endoftext|>


<commit_before>import csv

from django.core.management import BaseCommand

from corehq.apps.es import CaseES
from corehq.apps.es import filters

from dimagi.utils.chunked import chunked

CHILD_PROPERTIES = ['case_id', 'owner_id', 'opened_on', 'server_modified_on', 'name', 'indices', 'aadhar_number', 'dob', 'died']

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'hh_file',
            help="File path for household case output file",
        )
        parser.add_argument(
            'child_file',
            help="File path child case output file",
        )

    def handle(self, hh_file, child_file, **options):
        hh_cases = self._get_closed_hh_cases()
        with open(hh_file, 'w') as hh_csv, open(child_file, 'w') as child_csv:
            hh_writer = csv.writer(hh_csv)
            child_writer = csv.writer(child_csv)
            for cases in chunked(hh_cases.hits, 500):
                household_ids = []
                for hh in cases:
                    hh_writer.writerow([hh['case_id'], hh['date_closed']])
                    household_ids.append(hh['case_id'])
                child_cases = self._get_child_cases(household_ids)
                for child in child_cases.hits:
                    row = [child.get(prop, '') for prop in CHILD_PROPERTIES]
                    child_writer.writerow(row)

    def _get_closed_hh_cases(self):
        query = (CaseES()
                 .is_closed()
                 .domain('icds-cas')
                 .case_type('household')
                 .source(['case_id', 'date_closed'])
                 )
        return query.scroll()

    def _get_child_cases(self, household_ids):
        query = (CaseES()
                 .domain('icds-cas')
                 .case_type('person')
                 .source(CHILD_PROPERTIES)
                 .filter(
                    filters.nested('indices',
                                   filters.AND(
                                       filters.term("indices.referenced_id", household_ids),
                                       filters.term("indices.identifier", 'parent'))
                                   )
                    )
                )
        return query.run()
<commit_msg>adding progress bar<commit_after>import csv

from django.core.management import BaseCommand

from corehq.apps.es import CaseES
from corehq.apps.es import filters

from dimagi.utils.chunked import chunked

from corehq.util.log import with_progress_bar

CHILD_PROPERTIES = ['case_id', 'owner_id', 'opened_on', 'server_modified_on',
                    'name', 'indices', 'aadhar_number', 'dob', 'died']

class Command(BaseCommand):

    def add_arguments(self, parser):
        parser.add_argument(
            'hh_file',
            help="File path for household case output file",
        )
        parser.add_argument(
            'child_file',
            help="File path child case output file",
        )

    def handle(self, hh_file, child_file, **options):
        hh_cases = self._get_closed_hh_cases()
        with open(hh_file, 'w') as hh_csv, open(child_file, 'w') as child_csv:
            hh_writer = csv.writer(hh_csv)
            child_writer = csv.writer(child_csv)
            for cases in chunked(with_progress_bar(hh_cases.hits), 500):
                household_ids = []
                for hh in cases:
                    hh_writer.writerow([hh['case_id'], hh['date_closed']])
                    household_ids.append(hh['case_id'])
                child_cases = self._get_child_cases(household_ids)
                for child in child_cases.hits:
                    row = [child.get(prop, '') for prop in CHILD_PROPERTIES]
                    child_writer.writerow(row)

    def _get_closed_hh_cases(self):
        query = (CaseES()
                 .is_closed()
                 .domain('icds-cas')
                 .case_type('household')
                 .source(['case_id', 'date_closed'])
                 )
        return query.scroll()

    def _get_child_cases(self, household_ids):
        query = (CaseES()
                 .domain('icds-cas')
                 .case_type('person')
                 .source(CHILD_PROPERTIES)
                 .filter(
                    filters.nested('indices',
                                   filters.AND(
                                       filters.term("indices.referenced_id", household_ids),
                                       filters.term("indices.identifier", 'parent'))
                                   )
                    )
                )
        return query.run()
<|endoftext|>


<commit_before>from dataframe.dataframe import DataFrame
import numpy as np
from random import randint

# def _toy_loss(theta,y):
#     return (np.array([[0.5*((theta-y)**2).sum()]]))

# def _toy_gradient(theta,y):
#     return (theta-y)

# def _toy_loss(theta,y,fval=True,grad=True):
def toy(theta_df, X_df, y_df, fval=True,grad=True):
    if theta_df == None:
        return (y_df._row_index.keys(),y_df._col_index.keys())
    else:
        diff = theta_df.get_matrix()-y_df.get_matrix()
        f = (np.array([[0.5*(diff**2).sum()]]))
        g = diff
        return (f,g)

# # Purely for testing purposes...
# def toy(y_df):
#     def __toy_wrapper__(theta_df,i=None,batch_size=None):
#         return _toy_loss(theta_df.get_matrix(),y_df.get_matrix(readonly=True))
#     return __toy_wrapper__

def _multiclass_prob(theta,X):
    log_p = theta.dot(X.T)
    stable_log_p = log_p - log_p.max(axis=0)
    p = np.exp(stable_log_p)
    return np.divide(p,p.sum(axis=0))

# def _softmax_loss(theta,X,y,reg,fval=True,grad=True):
def softmax(theta_df,X_df,y_df,reg=0.01,fval=True,grad=True):
    if theta_df == None:
        rows = [str(k) for k in np.unique(y_df.get_matrix())]
        cols = X_df._col_index.keys()
        return rows,cols
    else:
        theta = theta_df.get_matrix()
        X = X_df.get_matrix(readonly=True)
        y = y_df.get_matrix(readonly=True)
        P = _multiclass_prob(theta,X)
        n = X.shape[0]
        if fval:
            cost = np.mean(np.log([ P[y[i,0],i] for i in np.arange(n)]))
            penalty = reg/2*np.power(theta,2).sum()
            f = -cost+penalty
        else:
            f = None
        if grad:
            P_minus = -P
            for k in range(len(y)):
                P_minus[y[k,0],k] += 1
            penalty = reg*theta
            total = np.zeros(theta.shape)
            for k in range(len(y)):
                total = total+np.kron(P_minus[:,k,None],X[k,:])
            g = -(total/len(y))+penalty
        else:
            g = None
        return (f,g)

# def softmax(X_df,y_df,reg):

#     def __softmax_wrapper__(theta_df,i=None,batch_size=None,fval=True,grad=True):
#         (n,m) = X_df.shape()
#         if i==None:
#             i = randint(0,n-1)
#         if batch_size==None:
#             batch_size=1
#         start = i
#         end = min(i+batch_size,n)
#         return _softmax_loss(theta_df.get_matrix(),
#                              X_df[start:end,:].get_matrix(readonly=True),
#                              y_df[start:end,:].get_matrix(readonly=True),
#                              reg,fval,grad)
#     return __softmax_wrapper__

# def _softmax_reg_loss(theta_df,X_df,y_df,reg):
#     theta = theta_df.get_matrix()
#     X = X_df.get_matrix()
#     y = y_df.get_matrix()
#     n = X.shape[0]
#     P = _multiclass_prob(theta,X)
#     cost = np.mean(np.log([ P[y[i],i] for i in np.arange(n)]))
#     penalty = reg/2*np.power(theta,2).sum()
#     return -cost+penalty

# def _softmax_reg_gradient(theta_df,X_df,y_df,reg):
#     theta = theta_df.get_matrix()
#     X = X_df.get_matrix()
#     y = y_df.get_matrix()
#     p = -_multiclass_prob(theta,X)
#     for k in range(len(y)):
#         p[y[k,0],k] += 1
#     penalty = reg*theta
#     total = np.zeros(theta.shape)
#     for k in range(len(y)):
#         total = total+np.kron(p[:,k,None],X[k,:])
#     return -(total/len(y))+penalty

# softmax_reg = (_softmax_reg_loss,_softmax_reg_gradient)<commit_msg>deleted old loss file<commit_after><|endoftext|>