Dex-prachi commited on
Commit
ff9c2f4
·
1 Parent(s): 43136fe

fix: OpenEnv spec_version + app field, stricter grader manifest for Phase2 validators

Browse files
Files changed (4) hide show
  1. README.md +2 -0
  2. openenv.yaml +26 -4
  3. src/code_assist_env.py +11 -0
  4. src/server.py +31 -4
README.md CHANGED
@@ -34,6 +34,8 @@ All graders are deterministic and return a float score in `[0.0, 1.0]`.
34
  | `/step` | POST | Take a step with an action |
35
  | `/state` | GET | Get the current state |
36
  | `/tasks` | GET | List all tasks with grader metadata |
 
 
37
  | `/health` | GET | Health check |
38
 
39
  ## Running Locally
 
34
  | `/step` | POST | Take a step with an action |
35
  | `/state` | GET | Get the current state |
36
  | `/tasks` | GET | List all tasks with grader metadata |
37
+ | `/manifest/tasks` | GET | Same manifest as `/tasks` (validator alias) |
38
+ | `/openenv.yaml` | GET | Raw `openenv.yaml` from the repo |
39
  | `/health` | GET | Health check |
40
 
41
  ## Running Locally
openenv.yaml CHANGED
@@ -1,8 +1,13 @@
1
- name: "CRA-Environment"
 
 
 
 
 
 
 
2
  version: "1.0.0"
3
  description: "Contextual Reinforcement Autocomplete — code completion with KG hints and deterministic per-task graders."
4
- entrypoint: "src.server:app"
5
- runtime: fastapi
6
  tags:
7
  - openenv
8
  - code-assist
@@ -10,51 +15,66 @@ tags:
10
 
11
  tasks:
12
  - id: syntax-line
 
13
  name: syntax-line
14
  description: "Finish the line so the module is valid Python (balanced parentheses)."
15
  difficulty: easy
16
  max_steps: 100
17
  score_range: [0.0, 1.0]
 
 
18
  grader: grade_syntax_line
19
  graders:
20
  - id: grade_syntax_line
21
  type: deterministic
 
22
  implementation: CodeAssistEnv._grade_syntax_line
23
 
24
  - id: import-fix
 
25
  name: import-fix
26
  description: "Add the missing import so the file runs."
27
  difficulty: medium
28
  max_steps: 100
29
  score_range: [0.0, 1.0]
 
 
30
  grader: grade_import_fix
31
  graders:
32
  - id: grade_import_fix
33
  type: deterministic
 
34
  implementation: CodeAssistEnv._grade_import_fix
35
 
36
  - id: docstring-stub
 
37
  name: docstring-stub
38
  description: "Give the function a docstring and a body with a return statement."
39
  difficulty: hard
40
  max_steps: 100
41
  score_range: [0.0, 1.0]
 
 
42
  grader: grade_docstring_stub
43
  graders:
44
  - id: grade_docstring_stub
45
  type: deterministic
46
- implementation: CodeAssistEnv._grade_docstring
 
47
 
48
  graders:
49
  - id: grade_syntax_line
50
  task_id: syntax-line
51
  type: deterministic
 
52
  - id: grade_import_fix
53
  task_id: import-fix
54
  type: deterministic
 
55
  - id: grade_docstring_stub
56
  task_id: docstring-stub
57
  type: deterministic
 
58
 
59
  observation_space:
60
  type: CodeObservation
@@ -77,6 +97,8 @@ endpoints:
77
  step: POST /step
78
  state: GET /state
79
  tasks: GET /tasks
 
 
80
 
81
  infrastructure:
82
  framework: FastAPI
 
1
+ # OpenEnv manifest — spec_version + app required for strict validators (see openenv-cli templates).
2
+ spec_version: 1
3
+ name: CRA-Environment
4
+ type: space
5
+ runtime: fastapi
6
+ app: src.server:app
7
+ port: 7860
8
+
9
  version: "1.0.0"
10
  description: "Contextual Reinforcement Autocomplete — code completion with KG hints and deterministic per-task graders."
 
 
11
  tags:
12
  - openenv
13
  - code-assist
 
15
 
16
  tasks:
17
  - id: syntax-line
18
+ task_id: syntax-line
19
  name: syntax-line
20
  description: "Finish the line so the module is valid Python (balanced parentheses)."
21
  difficulty: easy
22
  max_steps: 100
23
  score_range: [0.0, 1.0]
24
+ graded: true
25
+ programmatic_grader: true
26
  grader: grade_syntax_line
27
  graders:
28
  - id: grade_syntax_line
29
  type: deterministic
30
+ enabled: true
31
  implementation: CodeAssistEnv._grade_syntax_line
32
 
33
  - id: import-fix
34
+ task_id: import-fix
35
  name: import-fix
36
  description: "Add the missing import so the file runs."
37
  difficulty: medium
38
  max_steps: 100
39
  score_range: [0.0, 1.0]
40
+ graded: true
41
+ programmatic_grader: true
42
  grader: grade_import_fix
43
  graders:
44
  - id: grade_import_fix
45
  type: deterministic
46
+ enabled: true
47
  implementation: CodeAssistEnv._grade_import_fix
48
 
49
  - id: docstring-stub
50
+ task_id: docstring-stub
51
  name: docstring-stub
52
  description: "Give the function a docstring and a body with a return statement."
53
  difficulty: hard
54
  max_steps: 100
55
  score_range: [0.0, 1.0]
56
+ graded: true
57
+ programmatic_grader: true
58
  grader: grade_docstring_stub
59
  graders:
60
  - id: grade_docstring_stub
61
  type: deterministic
62
+ enabled: true
63
+ implementation: CodeAssistEnv._grade_docstring_stub
64
 
65
  graders:
66
  - id: grade_syntax_line
67
  task_id: syntax-line
68
  type: deterministic
69
+ enabled: true
70
  - id: grade_import_fix
71
  task_id: import-fix
72
  type: deterministic
73
+ enabled: true
74
  - id: grade_docstring_stub
75
  task_id: docstring-stub
76
  type: deterministic
77
+ enabled: true
78
 
79
  observation_space:
80
  type: CodeObservation
 
97
  step: POST /step
98
  state: GET /state
99
  tasks: GET /tasks
100
+ manifest_tasks: GET /manifest/tasks
101
+ openenv_yaml: GET /openenv.yaml
102
 
103
  infrastructure:
104
  framework: FastAPI
src/code_assist_env.py CHANGED
@@ -50,16 +50,20 @@ def graded_tasks_manifest() -> list[dict[str, Any]]:
50
  out.append(
51
  {
52
  "id": task_id,
 
53
  "name": task_id,
54
  "description": cfg["instruction"],
55
  "difficulty": cfg["difficulty"],
56
  "max_steps": MAX_STEPS,
57
  "score_range": [0.0, 1.0],
 
 
58
  "grader": grader,
59
  "graders": [
60
  {
61
  "id": grader,
62
  "type": "deterministic",
 
63
  "task_id": task_id,
64
  }
65
  ],
@@ -75,16 +79,19 @@ def graders_registry() -> list[dict[str, Any]]:
75
  "id": "grade_syntax_line",
76
  "task_id": "syntax-line",
77
  "type": "deterministic",
 
78
  },
79
  {
80
  "id": "grade_import_fix",
81
  "task_id": "import-fix",
82
  "type": "deterministic",
 
83
  },
84
  {
85
  "id": "grade_docstring_stub",
86
  "task_id": "docstring-stub",
87
  "type": "deterministic",
 
88
  },
89
  ]
90
 
@@ -268,6 +275,10 @@ class CodeAssistEnv(Environment[CodeAction, CodeObservation, CodeState]):
268
  pass
269
  return min(1.0, score)
270
 
 
 
 
 
271
  def _grade_docstring_stub(self):
272
  score = 0.0
273
  try:
 
50
  out.append(
51
  {
52
  "id": task_id,
53
+ "task_id": task_id,
54
  "name": task_id,
55
  "description": cfg["instruction"],
56
  "difficulty": cfg["difficulty"],
57
  "max_steps": MAX_STEPS,
58
  "score_range": [0.0, 1.0],
59
+ "graded": True,
60
+ "programmatic_grader": True,
61
  "grader": grader,
62
  "graders": [
63
  {
64
  "id": grader,
65
  "type": "deterministic",
66
+ "enabled": True,
67
  "task_id": task_id,
68
  }
69
  ],
 
79
  "id": "grade_syntax_line",
80
  "task_id": "syntax-line",
81
  "type": "deterministic",
82
+ "enabled": True,
83
  },
84
  {
85
  "id": "grade_import_fix",
86
  "task_id": "import-fix",
87
  "type": "deterministic",
88
+ "enabled": True,
89
  },
90
  {
91
  "id": "grade_docstring_stub",
92
  "task_id": "docstring-stub",
93
  "type": "deterministic",
94
+ "enabled": True,
95
  },
96
  ]
97
 
 
275
  pass
276
  return min(1.0, score)
277
 
278
+ def _grade_docstring(self) -> float:
279
+ """Alias for YAML `implementation: CodeAssistEnv._grade_docstring` compatibility."""
280
+ return self._grade_docstring_stub()
281
+
282
  def _grade_docstring_stub(self):
283
  score = 0.0
284
  try:
src/server.py CHANGED
@@ -12,7 +12,9 @@ from .code_assist_env import CodeAssistEnv
12
  from .models import CodeAction, CodeObservation
13
 
14
  _root = Path(__file__).parent
15
- _static = _root.parent / "static"
 
 
16
 
17
  app = create_fastapi_app(
18
  CodeAssistEnv,
@@ -45,9 +47,7 @@ async def predict(data: IDECpntext):
45
  return get_completion(data.content, data.cursor_offset)
46
 
47
 
48
- @app.get("/tasks", include_in_schema=True)
49
- async def list_graded_tasks():
50
- """Expose ≥3 graded tasks for automated submission validation."""
51
  from .code_assist_env import graded_tasks_manifest, graders_registry
52
 
53
  tasks = graded_tasks_manifest()
@@ -57,8 +57,35 @@ async def list_graded_tasks():
57
  "graders": graders,
58
  "task_count": len(tasks),
59
  "grader_count": len(graders),
 
 
 
 
 
60
  }
61
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
  if _static.is_dir():
63
  app.mount(
64
  "/static",
 
12
  from .models import CodeAction, CodeObservation
13
 
14
  _root = Path(__file__).parent
15
+ _repo_root = _root.parent
16
+ _static = _repo_root / "static"
17
+ _openenv_yaml = _repo_root / "openenv.yaml"
18
 
19
  app = create_fastapi_app(
20
  CodeAssistEnv,
 
47
  return get_completion(data.content, data.cursor_offset)
48
 
49
 
50
+ def _tasks_payload() -> dict:
 
 
51
  from .code_assist_env import graded_tasks_manifest, graders_registry
52
 
53
  tasks = graded_tasks_manifest()
 
57
  "graders": graders,
58
  "task_count": len(tasks),
59
  "grader_count": len(graders),
60
+ "tasks_with_graders": sum(
61
+ 1
62
+ for t in tasks
63
+ if t.get("grader") or (isinstance(t.get("graders"), list) and len(t["graders"]) > 0)
64
+ ),
65
  }
66
 
67
+
68
+ @app.get("/tasks", include_in_schema=True)
69
+ async def list_graded_tasks():
70
+ """Expose ≥3 graded tasks for automated submission validation."""
71
+ return _tasks_payload()
72
+
73
+
74
+ @app.get("/manifest/tasks", include_in_schema=True)
75
+ async def manifest_tasks_alias():
76
+ """Alternate path some validators probe for task/grader manifests."""
77
+ return _tasks_payload()
78
+
79
+
80
+ @app.get("/openenv.yaml", include_in_schema=False)
81
+ async def serve_openenv_yaml():
82
+ """Serve the repo manifest verbatim (some pipelines fetch this URL)."""
83
+ if not _openenv_yaml.is_file():
84
+ from fastapi import HTTPException
85
+
86
+ raise HTTPException(status_code=404, detail="openenv.yaml not found")
87
+ return FileResponse(_openenv_yaml, media_type="application/yaml")
88
+
89
  if _static.is_dir():
90
  app.mount(
91
  "/static",