Spaces:
Sleeping
Sleeping
Commit ·
ff9c2f4
1
Parent(s): 43136fe
fix: OpenEnv spec_version + app field, stricter grader manifest for Phase2 validators
Browse files- README.md +2 -0
- openenv.yaml +26 -4
- src/code_assist_env.py +11 -0
- src/server.py +31 -4
README.md
CHANGED
|
@@ -34,6 +34,8 @@ All graders are deterministic and return a float score in `[0.0, 1.0]`.
|
|
| 34 |
| `/step` | POST | Take a step with an action |
|
| 35 |
| `/state` | GET | Get the current state |
|
| 36 |
| `/tasks` | GET | List all tasks with grader metadata |
|
|
|
|
|
|
|
| 37 |
| `/health` | GET | Health check |
|
| 38 |
|
| 39 |
## Running Locally
|
|
|
|
| 34 |
| `/step` | POST | Take a step with an action |
|
| 35 |
| `/state` | GET | Get the current state |
|
| 36 |
| `/tasks` | GET | List all tasks with grader metadata |
|
| 37 |
+
| `/manifest/tasks` | GET | Same manifest as `/tasks` (validator alias) |
|
| 38 |
+
| `/openenv.yaml` | GET | Raw `openenv.yaml` from the repo |
|
| 39 |
| `/health` | GET | Health check |
|
| 40 |
|
| 41 |
## Running Locally
|
openenv.yaml
CHANGED
|
@@ -1,8 +1,13 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
version: "1.0.0"
|
| 3 |
description: "Contextual Reinforcement Autocomplete — code completion with KG hints and deterministic per-task graders."
|
| 4 |
-
entrypoint: "src.server:app"
|
| 5 |
-
runtime: fastapi
|
| 6 |
tags:
|
| 7 |
- openenv
|
| 8 |
- code-assist
|
|
@@ -10,51 +15,66 @@ tags:
|
|
| 10 |
|
| 11 |
tasks:
|
| 12 |
- id: syntax-line
|
|
|
|
| 13 |
name: syntax-line
|
| 14 |
description: "Finish the line so the module is valid Python (balanced parentheses)."
|
| 15 |
difficulty: easy
|
| 16 |
max_steps: 100
|
| 17 |
score_range: [0.0, 1.0]
|
|
|
|
|
|
|
| 18 |
grader: grade_syntax_line
|
| 19 |
graders:
|
| 20 |
- id: grade_syntax_line
|
| 21 |
type: deterministic
|
|
|
|
| 22 |
implementation: CodeAssistEnv._grade_syntax_line
|
| 23 |
|
| 24 |
- id: import-fix
|
|
|
|
| 25 |
name: import-fix
|
| 26 |
description: "Add the missing import so the file runs."
|
| 27 |
difficulty: medium
|
| 28 |
max_steps: 100
|
| 29 |
score_range: [0.0, 1.0]
|
|
|
|
|
|
|
| 30 |
grader: grade_import_fix
|
| 31 |
graders:
|
| 32 |
- id: grade_import_fix
|
| 33 |
type: deterministic
|
|
|
|
| 34 |
implementation: CodeAssistEnv._grade_import_fix
|
| 35 |
|
| 36 |
- id: docstring-stub
|
|
|
|
| 37 |
name: docstring-stub
|
| 38 |
description: "Give the function a docstring and a body with a return statement."
|
| 39 |
difficulty: hard
|
| 40 |
max_steps: 100
|
| 41 |
score_range: [0.0, 1.0]
|
|
|
|
|
|
|
| 42 |
grader: grade_docstring_stub
|
| 43 |
graders:
|
| 44 |
- id: grade_docstring_stub
|
| 45 |
type: deterministic
|
| 46 |
-
|
|
|
|
| 47 |
|
| 48 |
graders:
|
| 49 |
- id: grade_syntax_line
|
| 50 |
task_id: syntax-line
|
| 51 |
type: deterministic
|
|
|
|
| 52 |
- id: grade_import_fix
|
| 53 |
task_id: import-fix
|
| 54 |
type: deterministic
|
|
|
|
| 55 |
- id: grade_docstring_stub
|
| 56 |
task_id: docstring-stub
|
| 57 |
type: deterministic
|
|
|
|
| 58 |
|
| 59 |
observation_space:
|
| 60 |
type: CodeObservation
|
|
@@ -77,6 +97,8 @@ endpoints:
|
|
| 77 |
step: POST /step
|
| 78 |
state: GET /state
|
| 79 |
tasks: GET /tasks
|
|
|
|
|
|
|
| 80 |
|
| 81 |
infrastructure:
|
| 82 |
framework: FastAPI
|
|
|
|
| 1 |
+
# OpenEnv manifest — spec_version + app required for strict validators (see openenv-cli templates).
|
| 2 |
+
spec_version: 1
|
| 3 |
+
name: CRA-Environment
|
| 4 |
+
type: space
|
| 5 |
+
runtime: fastapi
|
| 6 |
+
app: src.server:app
|
| 7 |
+
port: 7860
|
| 8 |
+
|
| 9 |
version: "1.0.0"
|
| 10 |
description: "Contextual Reinforcement Autocomplete — code completion with KG hints and deterministic per-task graders."
|
|
|
|
|
|
|
| 11 |
tags:
|
| 12 |
- openenv
|
| 13 |
- code-assist
|
|
|
|
| 15 |
|
| 16 |
tasks:
|
| 17 |
- id: syntax-line
|
| 18 |
+
task_id: syntax-line
|
| 19 |
name: syntax-line
|
| 20 |
description: "Finish the line so the module is valid Python (balanced parentheses)."
|
| 21 |
difficulty: easy
|
| 22 |
max_steps: 100
|
| 23 |
score_range: [0.0, 1.0]
|
| 24 |
+
graded: true
|
| 25 |
+
programmatic_grader: true
|
| 26 |
grader: grade_syntax_line
|
| 27 |
graders:
|
| 28 |
- id: grade_syntax_line
|
| 29 |
type: deterministic
|
| 30 |
+
enabled: true
|
| 31 |
implementation: CodeAssistEnv._grade_syntax_line
|
| 32 |
|
| 33 |
- id: import-fix
|
| 34 |
+
task_id: import-fix
|
| 35 |
name: import-fix
|
| 36 |
description: "Add the missing import so the file runs."
|
| 37 |
difficulty: medium
|
| 38 |
max_steps: 100
|
| 39 |
score_range: [0.0, 1.0]
|
| 40 |
+
graded: true
|
| 41 |
+
programmatic_grader: true
|
| 42 |
grader: grade_import_fix
|
| 43 |
graders:
|
| 44 |
- id: grade_import_fix
|
| 45 |
type: deterministic
|
| 46 |
+
enabled: true
|
| 47 |
implementation: CodeAssistEnv._grade_import_fix
|
| 48 |
|
| 49 |
- id: docstring-stub
|
| 50 |
+
task_id: docstring-stub
|
| 51 |
name: docstring-stub
|
| 52 |
description: "Give the function a docstring and a body with a return statement."
|
| 53 |
difficulty: hard
|
| 54 |
max_steps: 100
|
| 55 |
score_range: [0.0, 1.0]
|
| 56 |
+
graded: true
|
| 57 |
+
programmatic_grader: true
|
| 58 |
grader: grade_docstring_stub
|
| 59 |
graders:
|
| 60 |
- id: grade_docstring_stub
|
| 61 |
type: deterministic
|
| 62 |
+
enabled: true
|
| 63 |
+
implementation: CodeAssistEnv._grade_docstring_stub
|
| 64 |
|
| 65 |
graders:
|
| 66 |
- id: grade_syntax_line
|
| 67 |
task_id: syntax-line
|
| 68 |
type: deterministic
|
| 69 |
+
enabled: true
|
| 70 |
- id: grade_import_fix
|
| 71 |
task_id: import-fix
|
| 72 |
type: deterministic
|
| 73 |
+
enabled: true
|
| 74 |
- id: grade_docstring_stub
|
| 75 |
task_id: docstring-stub
|
| 76 |
type: deterministic
|
| 77 |
+
enabled: true
|
| 78 |
|
| 79 |
observation_space:
|
| 80 |
type: CodeObservation
|
|
|
|
| 97 |
step: POST /step
|
| 98 |
state: GET /state
|
| 99 |
tasks: GET /tasks
|
| 100 |
+
manifest_tasks: GET /manifest/tasks
|
| 101 |
+
openenv_yaml: GET /openenv.yaml
|
| 102 |
|
| 103 |
infrastructure:
|
| 104 |
framework: FastAPI
|
src/code_assist_env.py
CHANGED
|
@@ -50,16 +50,20 @@ def graded_tasks_manifest() -> list[dict[str, Any]]:
|
|
| 50 |
out.append(
|
| 51 |
{
|
| 52 |
"id": task_id,
|
|
|
|
| 53 |
"name": task_id,
|
| 54 |
"description": cfg["instruction"],
|
| 55 |
"difficulty": cfg["difficulty"],
|
| 56 |
"max_steps": MAX_STEPS,
|
| 57 |
"score_range": [0.0, 1.0],
|
|
|
|
|
|
|
| 58 |
"grader": grader,
|
| 59 |
"graders": [
|
| 60 |
{
|
| 61 |
"id": grader,
|
| 62 |
"type": "deterministic",
|
|
|
|
| 63 |
"task_id": task_id,
|
| 64 |
}
|
| 65 |
],
|
|
@@ -75,16 +79,19 @@ def graders_registry() -> list[dict[str, Any]]:
|
|
| 75 |
"id": "grade_syntax_line",
|
| 76 |
"task_id": "syntax-line",
|
| 77 |
"type": "deterministic",
|
|
|
|
| 78 |
},
|
| 79 |
{
|
| 80 |
"id": "grade_import_fix",
|
| 81 |
"task_id": "import-fix",
|
| 82 |
"type": "deterministic",
|
|
|
|
| 83 |
},
|
| 84 |
{
|
| 85 |
"id": "grade_docstring_stub",
|
| 86 |
"task_id": "docstring-stub",
|
| 87 |
"type": "deterministic",
|
|
|
|
| 88 |
},
|
| 89 |
]
|
| 90 |
|
|
@@ -268,6 +275,10 @@ class CodeAssistEnv(Environment[CodeAction, CodeObservation, CodeState]):
|
|
| 268 |
pass
|
| 269 |
return min(1.0, score)
|
| 270 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
def _grade_docstring_stub(self):
|
| 272 |
score = 0.0
|
| 273 |
try:
|
|
|
|
| 50 |
out.append(
|
| 51 |
{
|
| 52 |
"id": task_id,
|
| 53 |
+
"task_id": task_id,
|
| 54 |
"name": task_id,
|
| 55 |
"description": cfg["instruction"],
|
| 56 |
"difficulty": cfg["difficulty"],
|
| 57 |
"max_steps": MAX_STEPS,
|
| 58 |
"score_range": [0.0, 1.0],
|
| 59 |
+
"graded": True,
|
| 60 |
+
"programmatic_grader": True,
|
| 61 |
"grader": grader,
|
| 62 |
"graders": [
|
| 63 |
{
|
| 64 |
"id": grader,
|
| 65 |
"type": "deterministic",
|
| 66 |
+
"enabled": True,
|
| 67 |
"task_id": task_id,
|
| 68 |
}
|
| 69 |
],
|
|
|
|
| 79 |
"id": "grade_syntax_line",
|
| 80 |
"task_id": "syntax-line",
|
| 81 |
"type": "deterministic",
|
| 82 |
+
"enabled": True,
|
| 83 |
},
|
| 84 |
{
|
| 85 |
"id": "grade_import_fix",
|
| 86 |
"task_id": "import-fix",
|
| 87 |
"type": "deterministic",
|
| 88 |
+
"enabled": True,
|
| 89 |
},
|
| 90 |
{
|
| 91 |
"id": "grade_docstring_stub",
|
| 92 |
"task_id": "docstring-stub",
|
| 93 |
"type": "deterministic",
|
| 94 |
+
"enabled": True,
|
| 95 |
},
|
| 96 |
]
|
| 97 |
|
|
|
|
| 275 |
pass
|
| 276 |
return min(1.0, score)
|
| 277 |
|
| 278 |
+
def _grade_docstring(self) -> float:
|
| 279 |
+
"""Alias for YAML `implementation: CodeAssistEnv._grade_docstring` compatibility."""
|
| 280 |
+
return self._grade_docstring_stub()
|
| 281 |
+
|
| 282 |
def _grade_docstring_stub(self):
|
| 283 |
score = 0.0
|
| 284 |
try:
|
src/server.py
CHANGED
|
@@ -12,7 +12,9 @@ from .code_assist_env import CodeAssistEnv
|
|
| 12 |
from .models import CodeAction, CodeObservation
|
| 13 |
|
| 14 |
_root = Path(__file__).parent
|
| 15 |
-
|
|
|
|
|
|
|
| 16 |
|
| 17 |
app = create_fastapi_app(
|
| 18 |
CodeAssistEnv,
|
|
@@ -45,9 +47,7 @@ async def predict(data: IDECpntext):
|
|
| 45 |
return get_completion(data.content, data.cursor_offset)
|
| 46 |
|
| 47 |
|
| 48 |
-
|
| 49 |
-
async def list_graded_tasks():
|
| 50 |
-
"""Expose ≥3 graded tasks for automated submission validation."""
|
| 51 |
from .code_assist_env import graded_tasks_manifest, graders_registry
|
| 52 |
|
| 53 |
tasks = graded_tasks_manifest()
|
|
@@ -57,8 +57,35 @@ async def list_graded_tasks():
|
|
| 57 |
"graders": graders,
|
| 58 |
"task_count": len(tasks),
|
| 59 |
"grader_count": len(graders),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
}
|
| 61 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 62 |
if _static.is_dir():
|
| 63 |
app.mount(
|
| 64 |
"/static",
|
|
|
|
| 12 |
from .models import CodeAction, CodeObservation
|
| 13 |
|
| 14 |
_root = Path(__file__).parent
|
| 15 |
+
_repo_root = _root.parent
|
| 16 |
+
_static = _repo_root / "static"
|
| 17 |
+
_openenv_yaml = _repo_root / "openenv.yaml"
|
| 18 |
|
| 19 |
app = create_fastapi_app(
|
| 20 |
CodeAssistEnv,
|
|
|
|
| 47 |
return get_completion(data.content, data.cursor_offset)
|
| 48 |
|
| 49 |
|
| 50 |
+
def _tasks_payload() -> dict:
|
|
|
|
|
|
|
| 51 |
from .code_assist_env import graded_tasks_manifest, graders_registry
|
| 52 |
|
| 53 |
tasks = graded_tasks_manifest()
|
|
|
|
| 57 |
"graders": graders,
|
| 58 |
"task_count": len(tasks),
|
| 59 |
"grader_count": len(graders),
|
| 60 |
+
"tasks_with_graders": sum(
|
| 61 |
+
1
|
| 62 |
+
for t in tasks
|
| 63 |
+
if t.get("grader") or (isinstance(t.get("graders"), list) and len(t["graders"]) > 0)
|
| 64 |
+
),
|
| 65 |
}
|
| 66 |
|
| 67 |
+
|
| 68 |
+
@app.get("/tasks", include_in_schema=True)
|
| 69 |
+
async def list_graded_tasks():
|
| 70 |
+
"""Expose ≥3 graded tasks for automated submission validation."""
|
| 71 |
+
return _tasks_payload()
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@app.get("/manifest/tasks", include_in_schema=True)
|
| 75 |
+
async def manifest_tasks_alias():
|
| 76 |
+
"""Alternate path some validators probe for task/grader manifests."""
|
| 77 |
+
return _tasks_payload()
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
@app.get("/openenv.yaml", include_in_schema=False)
|
| 81 |
+
async def serve_openenv_yaml():
|
| 82 |
+
"""Serve the repo manifest verbatim (some pipelines fetch this URL)."""
|
| 83 |
+
if not _openenv_yaml.is_file():
|
| 84 |
+
from fastapi import HTTPException
|
| 85 |
+
|
| 86 |
+
raise HTTPException(status_code=404, detail="openenv.yaml not found")
|
| 87 |
+
return FileResponse(_openenv_yaml, media_type="application/yaml")
|
| 88 |
+
|
| 89 |
if _static.is_dir():
|
| 90 |
app.mount(
|
| 91 |
"/static",
|