diff --git a/autograder/autograder.py b/autograder/autograder.py index 715eb6e..95ee6e8 100644 --- a/autograder/autograder.py +++ b/autograder/autograder.py @@ -137,7 +137,7 @@ def build_pipeline( Build the AutograderPipeline object based on configuration. Args: - template_name: Name of the template to use + template_name: Name of the template to use (string or list of strings) include_feedback: Whether to include feedback generation grading_criteria: Criteria configuration dictionary feedback_config: Configuration for feedback generation diff --git a/autograder/models/dataclass/structural_analysis_result.py b/autograder/models/dataclass/structural_analysis_result.py index c4143b1..c725e48 100644 --- a/autograder/models/dataclass/structural_analysis_result.py +++ b/autograder/models/dataclass/structural_analysis_result.py @@ -12,5 +12,9 @@ class StructuralAnalysisResult: Attributes: roots: A dictionary mapping filenames to their corresponding ast-grep root nodes. If a file could not be parsed, the value is None. + available: Whether structural analysis infrastructure was available and attempted. + reason: Optional reason explaining why analysis was unavailable/skipped. """ roots: Dict[str, Optional['SgRoot']] + available: bool = True + reason: Optional[str] = None diff --git a/autograder/models/pipeline_execution.py b/autograder/models/pipeline_execution.py index 0941eea..3e82aeb 100644 --- a/autograder/models/pipeline_execution.py +++ b/autograder/models/pipeline_execution.py @@ -99,14 +99,24 @@ def _require_step_data(self, step_name: StepName, artifact_name: str) -> Any: ) return step_result.data + def get_loaded_templates(self) -> List["Template"]: + """ + Retrieves the Grading Template objects loaded for this execution. + """ + data = self._require_step_data(StepName.LOAD_TEMPLATE, "templates") + if isinstance(data, list): + return cast(List["Template"], data) + return [cast("Template", data)] + def get_loaded_template(self) -> "Template": """ - Retrieves the Grading Template object loaded for this execution. + Retrieves the first Grading Template object loaded for this execution. + (Maintained for backward compatibility) """ - return cast( - "Template", - self._require_step_data(StepName.LOAD_TEMPLATE, "template"), - ) + templates = self.get_loaded_templates() + if not templates: + raise ValueError("No templates loaded in the pipeline.") + return templates[0] def get_built_criteria_tree(self) -> "CriteriaTree": """ diff --git a/autograder/services/criteria_tree_service.py b/autograder/services/criteria_tree_service.py index c3d5e53..35985be 100644 --- a/autograder/services/criteria_tree_service.py +++ b/autograder/services/criteria_tree_service.py @@ -27,26 +27,26 @@ class CriteriaTreeService: def __init__(self): self.logger = logging.getLogger("CriteriaTreeService") - self.__template = None + self.__templates: List[Template] = [] def build_tree( - self, criteria_config: CriteriaConfig, template: Template + self, criteria_config: CriteriaConfig, templates: List[Template] ) -> CriteriaTree: """ Build a complete criteria tree from validated configuration. Args: criteria_config: Validated criteria configuration - template: Template containing test functions + templates: List of templates containing test functions Returns: Complete CriteriaTree with embedded test functions Raises: - ValueError: If test function not found in template + ValueError: If test function not found in any template """ - self.__template = template + self.__templates = templates base_category = self.__parse_category("base", criteria_config.base) tree = CriteriaTree(base_category) @@ -88,10 +88,12 @@ def __parse_tests(self, test_configs: List[TestConfig]) -> List[TestNode]: return [self.__parse_test(test_item) for test_item in test_configs] def __find_test_function(self, name: str) -> Optional[TestFunction]: - try: - return self.__template.get_test(name) - except (AttributeError, KeyError): - return None + for template in self.__templates: + try: + return template.get_test(name) + except (AttributeError, KeyError): + continue + return None def __parse_test(self, config: TestConfig) -> TestNode: # Use technical 'type' for function lookup, falling back to 'name' for legacy support diff --git a/autograder/services/grader/criteria_grader.py b/autograder/services/grader/criteria_grader.py index e161888..ccdb69e 100644 --- a/autograder/services/grader/criteria_grader.py +++ b/autograder/services/grader/criteria_grader.py @@ -139,10 +139,14 @@ def process_test(self, test: TestNode) -> TestResultNode: ) test_params['program_command'] = resolved - # Inject the actual submission language so tests like forbidden_import - # always operate on the real language rather than a config-time guess. - if self.submission_language and 'submission_language' in test_params: - test_params['submission_language'] = self.submission_language + # Ensure submission_language is passed only once. + # Runtime language always takes precedence over config-specified language. + config_submission_language = test_params.pop('submission_language', None) + effective_submission_language = ( + self.submission_language + if self.submission_language is not None + else config_submission_language + ) test_result = test.test_function.execute( files=file_target, @@ -150,7 +154,7 @@ def process_test(self, test: TestNode) -> TestResultNode: locale=self.locale, pre_computed_results=self.pre_computed_results, structural_analysis=self.structural_analysis, - submission_language=self.submission_language, + submission_language=effective_submission_language, **test_params, ) return TestResultNode( diff --git a/autograder/services/template_library_service.py b/autograder/services/template_library_service.py index dcf15b1..300bfb7 100644 --- a/autograder/services/template_library_service.py +++ b/autograder/services/template_library_service.py @@ -13,6 +13,7 @@ from autograder.template_library.web_dev.template import WebDevTemplate from autograder.template_library.api_testing import ApiTestingTemplate from autograder.template_library.input_output import InputOutputTemplate +from autograder.template_library.static_analysis import StaticAnalysisTemplate logger = logging.getLogger(__name__) @@ -35,6 +36,7 @@ class TemplateLibraryService: "webdev": WebDevTemplate, "api": ApiTestingTemplate, "input_output": InputOutputTemplate, + "static_analysis": StaticAnalysisTemplate, } def __new__(cls): diff --git a/autograder/steps/build_tree_step.py b/autograder/steps/build_tree_step.py index 03aaf25..ca05c0e 100644 --- a/autograder/steps/build_tree_step.py +++ b/autograder/steps/build_tree_step.py @@ -43,11 +43,11 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: logger.info("Building criteria tree (external_user_id=%s)", pipeline_exec.submission.user_id) # Validate criteria configuration criteria_config = CriteriaConfig.from_dict(self._criteria_json) - template = pipeline_exec.get_loaded_template() - # Build the criteria tree with embedded test functions + templates = pipeline_exec.get_loaded_templates() + # Build the criteria tree with embedded test functions from multiple templates criteria_tree = self._criteria_tree_service.build_tree( criteria_config, - template + templates ) logger.info( "Criteria tree built successfully (external_user_id=%s)", diff --git a/autograder/steps/grade_step.py b/autograder/steps/grade_step.py index 1fce7ba..e6adf5d 100644 --- a/autograder/steps/grade_step.py +++ b/autograder/steps/grade_step.py @@ -41,13 +41,14 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: logger.info("Grading submission (external_user_id=%s)", pipeline_exec.submission.user_id) # If submission is sandboxed, feed grading template with container ref - template = pipeline_exec.get_loaded_template() + templates = pipeline_exec.get_loaded_templates() + requires_sandbox = any(t.requires_sandbox for t in templates) # Check if PRE_FLIGHT step was executed (only if setup_config was provided) sandbox = pipeline_exec.get_sandbox() - if not sandbox and template.requires_sandbox: - raise RuntimeError("Grading template requires a sandbox environment, but no sandbox was created") + if not sandbox and requires_sandbox: + raise RuntimeError("One or more grading templates require a sandbox environment, but no sandbox was created") criteria_tree = pipeline_exec.get_built_criteria_tree() diff --git a/autograder/steps/load_template_step.py b/autograder/steps/load_template_step.py index 60b8ade..a80b6f2 100644 --- a/autograder/steps/load_template_step.py +++ b/autograder/steps/load_template_step.py @@ -1,4 +1,5 @@ import logging +from typing import List, Union from autograder.models.dataclass.step_result import StepResult, StepName, StepStatus from autograder.models.pipeline_execution import PipelineExecution @@ -10,17 +11,43 @@ class TemplateLoaderStep(Step): """ - Step that loads a grading template, which contains test functions and helper code used for grading. - It can load either a built-in template from the library or a custom template provided by the user. - If the template is custom, it should be loaded in a sandboxed environment to ensure security and isolation. + Step that loads one or more grading templates, which contain test functions + and helper code used for grading. """ - def __init__(self, template_name: str, custom_template = None): + def __init__(self, template_name: Union[str, List[str], None], custom_template=None): """ Initialize the template loader step. """ - self._template_name = template_name + self._template_names = self._normalize_template_names(template_name) self._custom_template = custom_template self._template_service = TemplateLibraryService.get_instance() + if not self._custom_template and not self._template_names: + raise ValueError( + "template_name must contain at least one non-empty template identifier." + ) + + @staticmethod + def _normalize_template_names(template_name: Union[str, List[str], None]) -> List[str]: + if template_name is None: + return [] + + if isinstance(template_name, str): + raw_names = template_name.split(",") + elif isinstance(template_name, list): + raw_names = template_name + else: + raise ValueError( + "template_name must be a comma-separated string or a list of strings." + ) + + normalized: List[str] = [] + for name in raw_names: + if not isinstance(name, str): + raise ValueError("template_name entries must be strings.") + stripped = name.strip() + if stripped: + normalized.append(stripped) + return normalized @property def step_name(self) -> StepName: @@ -28,31 +55,33 @@ def step_name(self) -> StepName: def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: """ - Load the grading template, either built-in or custom, and return it as part of the step result. + Load the grading templates and return them as part of the step result. """ + templates = [] + if self._custom_template: logger.info("Loading custom template (external_user_id=%s)", pipeline_exec.submission.user_id) - template = self._template_service.load_custom_template(self._custom_template) #TODO: Implement Custom Template Loading with Sandboxed Env + template = self._template_service.load_custom_template(self._custom_template) + templates.append(template) else: - logger.info( - "Loading built-in template: template=%s (external_user_id=%s)", - self._template_name, - pipeline_exec.submission.user_id, - ) - template = self._template_service.load_builtin_template(self._template_name) # Load built-in template similar to custom to avoid code duplication + for name in self._template_names: + logger.info( + "Loading built-in template: template=%s (external_user_id=%s)", + name, + pipeline_exec.submission.user_id, + ) + template = self._template_service.load_builtin_template(name) + templates.append(template) logger.info( - "Template loaded successfully: template=%s (external_user_id=%s)", - self._template_name, + "Templates loaded successfully: count=%d (external_user_id=%s)", + len(templates), pipeline_exec.submission.user_id, ) return pipeline_exec.add_step_result( StepResult( step=StepName.LOAD_TEMPLATE, - data=template, + data=templates, status=StepStatus.SUCCESS ) ) - - - diff --git a/autograder/steps/sandbox_step.py b/autograder/steps/sandbox_step.py index 4c1c89c..df421c0 100644 --- a/autograder/steps/sandbox_step.py +++ b/autograder/steps/sandbox_step.py @@ -36,10 +36,12 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: Returns: PipelineExecution with updated sandbox and step result. """ - grading_template = pipeline_exec.get_loaded_template() + grading_templates = pipeline_exec.get_loaded_templates() - if not grading_template.requires_sandbox: - logger.info("Template does not require a sandbox. Skipping SandboxStep.") + requires_sandbox = any(t.requires_sandbox for t in grading_templates) + + if not requires_sandbox: + logger.info("None of the templates require a sandbox. Skipping SandboxStep.") return pipeline_exec.add_step_result(StepResult.success(self.step_name, None)) logger.info("Creating sandbox for submission (external_user_id=%s)", pipeline_exec.submission.user_id) diff --git a/autograder/steps/structural_analysis_step.py b/autograder/steps/structural_analysis_step.py index aa49845..e530e71 100644 --- a/autograder/steps/structural_analysis_step.py +++ b/autograder/steps/structural_analysis_step.py @@ -27,19 +27,46 @@ def step_name(self) -> StepName: def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: submission = pipeline_exec.submission language = submission.language - + if not language: logger.warning("No language specified for submission; skipping structural analysis.") - return pipeline_exec.add_step_result(StepResult.success(self.step_name, StructuralAnalysisResult(roots={}))) + return pipeline_exec.add_step_result( + StepResult.success( + self.step_name, + StructuralAnalysisResult( + roots={}, + available=False, + reason="missing_submission_language", + ), + ) + ) if SgRoot is None: - logger.error("ast-grep-py is not installed; structural analysis will be skipped.") - return pipeline_exec.add_step_result(StepResult.fail(self.step_name, "ast-grep-py not installed")) + logger.warning("ast-grep-py is not installed; skipping structural analysis.") + return pipeline_exec.add_step_result( + StepResult.success( + self.step_name, + StructuralAnalysisResult( + roots={}, + available=False, + reason="ast_grep_unavailable", + ), + ) + ) ast_grep_lang = self._map_language(language) if not ast_grep_lang: logger.warning(f"Language {language.value} is not supported by ast-grep; skipping.") - return pipeline_exec.add_step_result(StepResult.success(self.step_name, StructuralAnalysisResult(roots={}))) + return pipeline_exec.add_step_result( + StepResult.success( + self.step_name, + StructuralAnalysisResult( + roots={}, + available=False, + reason=f"unsupported_language:{language.value}", + ), + ) + ) roots: Dict[str, Optional[SgRoot]] = {} for filename, sub_file in submission.submission_files.items(): @@ -53,7 +80,7 @@ def _execute(self, pipeline_exec: PipelineExecution) -> PipelineExecution: logger.warning(f"Failed to parse {filename} with ast-grep: {e}") roots[filename] = None - result = StructuralAnalysisResult(roots=roots) + result = StructuralAnalysisResult(roots=roots, available=True) return pipeline_exec.add_step_result(StepResult.success(self.step_name, result)) def _map_language(self, language: Language) -> Optional[str]: diff --git a/autograder/template_library/input_output.py b/autograder/template_library/input_output.py index e5c98e2..d8c64df 100644 --- a/autograder/template_library/input_output.py +++ b/autograder/template_library/input_output.py @@ -357,344 +357,6 @@ def _extract_and_compare(self, sandbox: SandboxContainer, path: str, expected: s path=path, expected=expected, actual=actual)) -# =============================================================== -# TestFunction for Forbidden Import Detection -# =============================================================== - -class ForbiddenImportTest(TestFunction): - """ - Tests that a submission does NOT import any of the specified forbidden libraries. - - Performs static analysis on submission file contents using language-aware - regex patterns. Supports Python, Java, JavaScript/Node, C and C++. - """ - - # Language-specific regex builders: each returns a compiled pattern - # that matches an import of the given library name. - IMPORT_PATTERNS = { - Language.PYTHON: [ - # import lib / import lib as x / import lib.sub - r'^\s*import\s+{lib}\b', - # from lib import ... / from lib.sub import ... - r'^\s*from\s+{lib}\b', - ], - Language.JAVA: [ - # import pkg.Class; / import static pkg.Class.method; - r'^\s*import\s+(?:static\s+)?{lib}\b', - ], - Language.NODE: [ - # require('lib') / require("lib") - r"\brequire\s*\(\s*['\"]{{lib}}['\"]\s*\)", - # import ... from 'lib' / import 'lib' - r'^\s*import\s+.*?[\'"]{{lib}}[\'"]', - ], - Language.CPP: [ - # #include / #include / #include "lib..." - r'^\s*#\s*include\s*[<"]{lib}[/\.>"]', - ], - Language.C: [ - r'^\s*#\s*include\s*[<"]{lib}[/\.>"]', - ], - } - - @property - def name(self): - return "forbidden_import" - - @property - def description(self): - return t("io.forbidden_import.description") - - @property - def required_file(self): - """Returns the name of the file required for this test, or None if not applicable.""" - return None - - @property - def parameter_description(self): - return [ - ParamDescription( - "forbidden_imports", - t("io.forbidden_import.params.libraries"), - "list of strings" - ), - ParamDescription( - "submission_language", - t("io.forbidden_import.params.language"), - "string or Language enum" - ), - ] - - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - - def _build_patterns(self, library: str, language: Language) -> List[re.Pattern]: - """Return compiled regex patterns for detecting *library* in *language*.""" - templates = self.IMPORT_PATTERNS.get(language, []) - compiled: List[re.Pattern] = [] - for tmpl in templates: - # Support both {lib} and {{lib}} placeholders (Node patterns use - # double-braces to survive the first .format call on the class body). - raw = tmpl.replace('{{lib}}', library).replace('{lib}', re.escape(library)) - compiled.append(re.compile(raw, re.MULTILINE)) - return compiled - - def _scan_file(self, content: str, forbidden: List[str], - language: Language) -> List[str]: - """ - Scan *content* for any forbidden imports. - - Returns a list of human-readable violation strings. - """ - violations: List[str] = [] - for lib in forbidden: - patterns = self._build_patterns(lib, language) - for pattern in patterns: - match = pattern.search(content) - if match: - violations.append(lib) - break # one match per library is enough - return violations - - @staticmethod - def _resolve_language(submission_language=None) -> Optional[Language]: - """Resolve a raw language value into a Language enum member.""" - if submission_language is None: - return None - if isinstance(submission_language, Language): - return submission_language - # Accept string values like "python", "java", etc. - for lang in Language: - if lang.value == str(submission_language).lower(): - return lang - return None - - # ------------------------------------------------------------------ - # Execute - # ------------------------------------------------------------------ - - def execute(self, files: Optional[List[SubmissionFile]], sandbox: Optional[SandboxContainer], - *args, forbidden_imports: List[str] = None, - submission_language=None, **kwargs) -> TestResult: - """ - Scan every submission file for forbidden imports. - - Returns score 100 if no forbidden imports are found, 0 otherwise. - """ - locale = kwargs.get("locale") - if not forbidden_imports: - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_import.report.no_imports", locale=locale) - ) - - language = self._resolve_language(submission_language) - - if language is None: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.forbidden_import.report.no_lang", locale=locale) - ) - - if not files: - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_import.report.no_files", locale=locale) - ) - - all_violations: List[str] = [] - for submission_file in files: - found = self._scan_file( - submission_file.content, forbidden_imports, language - ) - for lib in found: - all_violations.append( - t("io.forbidden_import.report.violation", locale=locale, lib=lib, file=submission_file.filename) - ) - - if all_violations: - details = "\n".join(all_violations) - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.forbidden_import.report.failure", locale=locale, details=details) - ) - - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_import.report.success", locale=locale) - ) - - -# =============================================================== -# TestFunction for Forbidden Keyword/Construct Detection -# =============================================================== - -class ForbiddenKeywordConfig(BaseModel): - """Configuration schema for ForbiddenKeywordTest.""" - forbidden_keywords: List[str] = Field(default_factory=list) - custom_ast_grep_rules: List[Dict[str, Any]] = Field(default_factory=list) - -class ForbiddenKeywordTest(TestFunction): - """ - Tests that a submission does NOT use any of the specified forbidden - keywords or language constructs. - - Uses ast-grep for structural analysis, which accurately detects - constructs even when they are not at the start of a line and - correctly ignores comments and string literals. - """ - - # Mapping of high-level keywords to language-specific ast-grep rules. - PREDEFINED_RULES: Dict[Language, Dict[str, Dict[str, Any]]] = { - Language.PYTHON: { - "for_loop": {"kind": "for_statement"}, - "while_loop": {"kind": "while_statement"}, - "eval_call": {"pattern": "eval($$$)"}, - "exec_call": {"pattern": "exec($$$)"}, - }, - Language.JAVA: { - "for_loop": {"kind": "for_statement"}, - "while_loop": {"kind": "while_statement"}, - }, - Language.NODE: { - "for_loop": {"kind": "for_statement"}, - "while_loop": {"kind": "while_statement"}, - "eval_call": {"pattern": "eval($$$)"}, - }, - Language.CPP: { - "for_loop": {"kind": "for_statement"}, - "while_loop": {"kind": "while_statement"}, - "do_while_loop": {"kind": "do_statement"}, - }, - Language.C: { - "for_loop": {"kind": "for_statement"}, - "while_loop": {"kind": "while_statement"}, - "do_while_loop": {"kind": "do_statement"}, - }, - } - - @property - def name(self): - return "forbidden_keyword" - - @property - def description(self): - return t("io.forbidden_keyword.description") - - @property - def parameter_description(self): - return [ - ParamDescription("forbidden_keywords", t("io.forbidden_keyword.params.keywords"), "list of strings"), - ParamDescription("custom_ast_grep_rules", t("io.forbidden_keyword.params.custom_rules"), "list of dicts"), - ] - - @property - def config_schema(self) -> Type[BaseModel]: - """Optional Pydantic model to validate test parameters during tree building.""" - return ForbiddenKeywordConfig - - def execute(self, files: Optional[List[SubmissionFile]], sandbox: Optional[SandboxContainer], - *args, forbidden_keywords: List[str] = None, - custom_ast_grep_rules: List[Dict[str, Any]] = None, - structural_analysis: Optional[StructuralAnalysisResult] = None, - submission_language: Optional[Language] = None, - **kwargs) -> TestResult: - """ - Scan target files for forbidden keywords or structural patterns. - """ - locale = kwargs.get("locale") - forbidden_keywords = forbidden_keywords or [] - custom_ast_grep_rules = custom_ast_grep_rules or [] - - if not forbidden_keywords and not custom_ast_grep_rules: - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_keyword.report.no_rules", locale=locale) - ) - - if structural_analysis is None: - # Fallback if structural analysis was not executed or failed - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.forbidden_keyword.report.no_analysis", locale=locale) - ) - - if submission_language is None: - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.forbidden_keyword.report.no_lang", locale=locale) - ) - - # Consolidate rules to run - active_rules: List[Dict[str, Any]] = list(custom_ast_grep_rules) - lang_predefined = self.PREDEFINED_RULES.get(submission_language, {}) - - for kw in forbidden_keywords: - if kw in lang_predefined: - active_rules.append(lang_predefined[kw]) - else: - # If a keyword is not predefined for this language, we skip it - # but might want to log it or report it as an internal warning. - pass - - if not active_rules: - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_keyword.report.success", locale=locale) - ) - - all_violations: List[str] = [] - - # Only check files passed to this test - if not files: - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_keyword.report.no_files", locale=locale) - ) - - for sub_file in files: - root = structural_analysis.roots.get(sub_file.filename) - if root is None: - continue - - for rule in active_rules: - matches = root.root().find_all(**rule) - if matches: - # Provide some feedback about what was found - rule_desc = rule.get("kind") or rule.get("pattern") or str(rule) - all_violations.append( - t("io.forbidden_keyword.report.violation", - locale=locale, - rule=rule_desc, - file=sub_file.filename) - ) - - if all_violations: - details = "\n".join(all_violations) - return TestResult( - test_name=self.name, - score=0.0, - report=t("io.forbidden_keyword.report.failure", locale=locale, details=details) - ) - - return TestResult( - test_name=self.name, - score=100.0, - report=t("io.forbidden_keyword.report.success", locale=locale) - ) - - class InputOutputTemplate(Template): """ A template for command-line I/O assignments. It uses the SandboxExecutor @@ -708,8 +370,6 @@ def __init__(self): "expect_output": ExpectOutputTest(), "dont_fail": DontFailTest(), "expect_file_artifact": ExpectFileArtifactTest(), - "forbidden_import": ForbiddenImportTest(), - "forbidden_keyword": ForbiddenKeywordTest(), } @property diff --git a/autograder/template_library/static_analysis.py b/autograder/template_library/static_analysis.py new file mode 100644 index 0000000..f6426dc --- /dev/null +++ b/autograder/template_library/static_analysis.py @@ -0,0 +1,365 @@ +import logging +from typing import List, Optional, Dict, Any, Type +from pydantic import BaseModel, Field + +from autograder.models.abstract.template import Template +from autograder.models.abstract.test_function import TestFunction +from autograder.models.dataclass.param_description import ParamDescription +from autograder.models.dataclass.submission import SubmissionFile +from autograder.models.dataclass.test_result import TestResult +from autograder.models.dataclass.structural_analysis_result import StructuralAnalysisResult +from autograder.translations import t +from sandbox_manager.sandbox_container import SandboxContainer +from sandbox_manager.models.sandbox_models import Language +import re + +# =============================================================== +# TestFunction for Forbidden Import Detection +# =============================================================== + +class ForbiddenImportTest(TestFunction): + """ + Tests that a submission does NOT import any of the specified forbidden libraries. + + Performs static analysis on submission file contents using language-aware + regex patterns. Supports Python, Java, JavaScript/Node, C and C++. + """ + + # Language-specific regex builders: each returns a compiled pattern + # that matches an import of the given library name. + IMPORT_PATTERNS = { + Language.PYTHON: [ + # import lib / import lib as x / import lib.sub + r'^\s*import\s+{lib}\b', + # from lib import ... / from lib.sub import ... + r'^\s*from\s+{lib}\b', + ], + Language.JAVA: [ + # import pkg.Class; / import static pkg.Class.method; + r'^\s*import\s+(?:static\s+)?{lib}\b', + ], + Language.NODE: [ + # require('lib') / require("lib") + r"\brequire\s*\(\s*['\"]{{lib}}['\"]\s*\)", + # import ... from 'lib' / import 'lib' + r'^\s*import\s+.*?[\'"]{{lib}}[\'"]', + ], + Language.CPP: [ + # #include / #include / #include "lib..." + r'^\s*#\s*include\s*[<"]{lib}[/\.>"]', + ], + Language.C: [ + r'^\s*#\s*include\s*[<"]{lib}[/\.>"]', + ], + } + + @property + def name(self): + return "forbidden_import" + + @property + def description(self): + return t("static_analysis.forbidden_import.description") + + @property + def required_file(self): + return None + + @property + def parameter_description(self): + return [ + ParamDescription( + "forbidden_imports", + t("static_analysis.forbidden_import.params.libraries"), + "list of strings" + ), + ParamDescription( + "submission_language", + t("static_analysis.forbidden_import.params.language"), + "string or Language enum" + ), + ] + + def _build_patterns(self, library: str, language: Language) -> List[re.Pattern]: + templates = self.IMPORT_PATTERNS.get(language, []) + compiled: List[re.Pattern] = [] + escaped_library = re.escape(library) + for tmpl in templates: + raw = tmpl.replace('{{lib}}', escaped_library).replace('{lib}', escaped_library) + compiled.append(re.compile(raw, re.MULTILINE)) + return compiled + + def _scan_file(self, content: str, forbidden: List[str], + language: Language) -> List[str]: + violations: List[str] = [] + for lib in forbidden: + patterns = self._build_patterns(lib, language) + for pattern in patterns: + match = pattern.search(content) + if match: + violations.append(lib) + break + return violations + + @staticmethod + def _resolve_language(submission_language=None) -> Optional[Language]: + if submission_language is None: + return None + if isinstance(submission_language, Language): + return submission_language + for lang in Language: + if lang.value == str(submission_language).lower(): + return lang + return None + + def execute(self, files: Optional[List[SubmissionFile]], sandbox: Optional[SandboxContainer], + *args, forbidden_imports: List[str] = None, + submission_language=None, **kwargs) -> TestResult: + locale = kwargs.get("locale") + if not forbidden_imports: + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_import.report.no_imports", locale=locale) + ) + + language = self._resolve_language(submission_language) + + if language is None: + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_import.report.no_lang", locale=locale) + ) + + if not files: + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_import.report.no_files", locale=locale) + ) + + all_violations: List[str] = [] + for submission_file in files: + found = self._scan_file( + submission_file.content, forbidden_imports, language + ) + for lib in found: + all_violations.append( + t("static_analysis.forbidden_import.report.violation", locale=locale, lib=lib, file=submission_file.filename) + ) + + if all_violations: + details = "\n".join(all_violations) + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_import.report.failure", locale=locale, details=details) + ) + + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_import.report.success", locale=locale) + ) + + +# =============================================================== +# TestFunction for Forbidden Keyword/Construct Detection +# =============================================================== + +class ForbiddenKeywordConfig(BaseModel): + forbidden_keywords: List[str] = Field(default_factory=list) + custom_ast_grep_rules: List[Dict[str, Any]] = Field(default_factory=list) + +class ForbiddenKeywordTest(TestFunction): + """ + Tests that a submission does NOT use any of the specified forbidden + keywords or language constructs. + """ + + PREDEFINED_RULES: Dict[Language, Dict[str, Dict[str, Any]]] = { + Language.PYTHON: { + "for_loop": {"kind": "for_statement"}, + "while_loop": {"kind": "while_statement"}, + "eval_call": {"pattern": "eval($$$)"}, + "exec_call": {"pattern": "exec($$$)"}, + }, + Language.JAVA: { + "for_loop": {"kind": "for_statement"}, + "while_loop": {"kind": "while_statement"}, + }, + Language.NODE: { + "for_loop": {"kind": "for_statement"}, + "while_loop": {"kind": "while_statement"}, + "eval_call": {"pattern": "eval($$$)"}, + }, + Language.CPP: { + "for_loop": {"kind": "for_statement"}, + "while_loop": {"kind": "while_statement"}, + "do_while_loop": {"kind": "do_statement"}, + }, + Language.C: { + "for_loop": {"kind": "for_statement"}, + "while_loop": {"kind": "while_statement"}, + "do_while_loop": {"kind": "do_statement"}, + }, + } + + @property + def name(self): + return "forbidden_keyword" + + @property + def description(self): + return t("static_analysis.forbidden_keyword.description") + + @property + def parameter_description(self): + return [ + ParamDescription("forbidden_keywords", t("static_analysis.forbidden_keyword.params.keywords"), "list of strings"), + ParamDescription("custom_ast_grep_rules", t("static_analysis.forbidden_keyword.params.custom_rules"), "list of dicts"), + ] + + @property + def config_schema(self) -> Type[BaseModel]: + return ForbiddenKeywordConfig + + def execute(self, files: Optional[List[SubmissionFile]], sandbox: Optional[SandboxContainer], + *args, forbidden_keywords: List[str] = None, + custom_ast_grep_rules: List[Dict[str, Any]] = None, + structural_analysis: Optional[StructuralAnalysisResult] = None, + submission_language: Optional[Language] = None, + **kwargs) -> TestResult: + locale = kwargs.get("locale") + forbidden_keywords = forbidden_keywords or [] + custom_ast_grep_rules = custom_ast_grep_rules or [] + + if not forbidden_keywords and not custom_ast_grep_rules: + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_keyword.report.no_rules", locale=locale) + ) + + if structural_analysis is None or not structural_analysis.available: + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_keyword.report.no_analysis", locale=locale) + ) + + if submission_language is None: + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_keyword.report.no_lang", locale=locale) + ) + + active_rules: List[Dict[str, Any]] = list(custom_ast_grep_rules) + lang_predefined = self.PREDEFINED_RULES.get(submission_language, {}) + + for kw in forbidden_keywords: + if kw in lang_predefined: + active_rules.append(lang_predefined[kw]) + + if not active_rules: + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_keyword.report.success", locale=locale) + ) + + all_violations: List[str] = [] + + if not files: + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_keyword.report.no_files", locale=locale) + ) + + if not structural_analysis.roots: + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_keyword.report.no_analysis", locale=locale) + ) + + missing_roots = [ + sub_file.filename + for sub_file in files + if sub_file.filename not in structural_analysis.roots + or structural_analysis.roots[sub_file.filename] is None + ] + if missing_roots: + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_keyword.report.no_analysis", locale=locale) + ) + + for sub_file in files: + root = structural_analysis.roots.get(sub_file.filename) + if root is None: + continue + + for rule in active_rules: + matches = root.root().find_all(**rule) + if matches: + rule_desc = rule.get("kind") or rule.get("pattern") or str(rule) + all_violations.append( + t("static_analysis.forbidden_keyword.report.violation", + locale=locale, + rule=rule_desc, + file=sub_file.filename) + ) + + if all_violations: + details = "\n".join(all_violations) + return TestResult( + test_name=self.name, + score=0.0, + report=t("static_analysis.forbidden_keyword.report.failure", locale=locale, details=details) + ) + + return TestResult( + test_name=self.name, + score=100.0, + report=t("static_analysis.forbidden_keyword.report.success", locale=locale) + ) + + +class StaticAnalysisTemplate(Template): + """ + A template for static analysis of code submissions. + """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + self.tests = { + "forbidden_import": ForbiddenImportTest(), + "forbidden_keyword": ForbiddenKeywordTest(), + } + + @property + def template_name(self): + return t("static_analysis.template.name") + + @property + def template_description(self): + return t("static_analysis.template.description") + + @property + def requires_sandbox(self) -> bool: + # Static analysis tests usually don't require execution, + # but structural analysis (ast-grep) might need a sandbox in some future versions. + # Currently, they run on the server side using the files content. + return False + + def get_test(self, name: str) -> TestFunction: + test_function = self.tests.get(name) + if not test_function: + raise AttributeError(f"Test '{name}' not found in the '{self.template_name}' template.") + return test_function diff --git a/autograder/translations/en.json b/autograder/translations/en.json index 830256c..212c331 100644 --- a/autograder/translations/en.json +++ b/autograder/translations/en.json @@ -449,25 +449,6 @@ "success": "Success: Program executed without errors." } }, - "forbidden_import": { - "description": "Statically analyzes submission files to check if any forbidden libraries were imported. Penalizes the submission if any forbidden imports are found.", - "params": { - "libraries": "List of library/module names whose import is forbidden.", - "language": "Submission language (e.g., 'python', 'java'). Required to identify correct import patterns." - }, - "report": { - "no_imports": "No forbidden imports specified — nothing to check.", - "no_lang": "FAILURE: Submission language is not defined. Cannot check for forbidden imports.", - "no_files": "No submission files to analyze.", - "violation": " - '{lib}' found in '{file}'", - "failure": "FAILURE: Forbidden imports detected:\n{details}", - "success": "No forbidden imports found." - } - }, - "template": { - "name": "Input/Output", - "description": "A template for evaluating assignments based on command-line input and output." - }, "expect_file_artifact": { "description": "Executes the student program, then extracts a generated file from the sandbox and validates its content against expected values.", "params": { @@ -488,6 +469,27 @@ "invalid_regex": "CONFIGURATION ERROR: Invalid regex pattern.\nDetails: {error}" } }, + "template": { + "name": "Input/Output", + "description": "A template for evaluating assignments based on command-line input and output." + } + }, + "static_analysis": { + "forbidden_import": { + "description": "Statically analyzes submission files to check if any forbidden libraries were imported. Penalizes the submission if any forbidden imports are found.", + "params": { + "libraries": "List of library/module names whose import is forbidden.", + "language": "Submission language (e.g., 'python', 'java'). Required to identify correct import patterns." + }, + "report": { + "no_imports": "No forbidden imports specified — nothing to check.", + "no_lang": "FAILURE: Submission language is not defined. Cannot check for forbidden imports.", + "no_files": "No submission files to analyze.", + "violation": " - '{lib}' found in '{file}'", + "failure": "FAILURE: Forbidden imports detected:\n{details}", + "success": "No forbidden imports found." + } + }, "forbidden_keyword": { "description": "Statically analyzes submission files using structural analysis to detect forbidden language constructs (like loops or eval calls).", "params": { @@ -503,6 +505,10 @@ "failure": "FAILURE: Forbidden language constructs detected:\n{details}", "success": "No forbidden language constructs found." } + }, + "template": { + "name": "Static Analysis", + "description": "A template for evaluating assignments through static and structural code analysis." } }, "api_testing": { @@ -542,4 +548,4 @@ "step_interrupted": "An unexpected error occurred during step execution: {error}" } } -} \ No newline at end of file +} diff --git a/autograder/translations/pt_br.json b/autograder/translations/pt_br.json index 7a95df3..df57c02 100644 --- a/autograder/translations/pt_br.json +++ b/autograder/translations/pt_br.json @@ -444,30 +444,11 @@ } }, "dont_fail": { - "description": "Executes the student program with specific input and verifies it doesn't fail (no crash, no timeout).", + "description": "Executa o programa do aluno com entrada específica e verifica se não falha (sem crash, sem timeout).", "report": { "success": "Sucesso: O programa foi executado sem erros." } }, - "forbidden_import": { - "description": "Analisa estaticamente os arquivos de submissão para verificar se alguma biblioteca proibida foi importada. Penaliza a submissão caso alguma importação proibida seja encontrada.", - "params": { - "libraries": "Lista de nomes de bibliotecas/módulos cuja importação é proibida.", - "language": "Linguagem da submissão (ex: 'python', 'java'). Necessária para identificar padrões de importação corretos." - }, - "report": { - "no_imports": "Nenhuma importação proibida especificada — nada para verificar.", - "no_lang": "FALHA: A linguagem da submissão não foi definida. Não é possível verificar importações proibidas.", - "no_files": "Nenhum arquivo de submissão para analisar.", - "violation": " - '{lib}' encontrado em '{file}'", - "failure": "FALHA: Importações proibidas detectadas:\n{details}", - "success": "Nenhuma importação proibida encontrada." - } - }, - "template": { - "name": "Entrada/Saída (I/O)", - "description": "Um modelo para avaliar trabalhos com base na entrada e saída de linha de comando." - }, "expect_file_artifact": { "description": "Executa o programa do aluno, extrai um arquivo gerado do sandbox e valida seu conteúdo contra os valores esperados.", "params": { @@ -488,6 +469,27 @@ "invalid_regex": "ERRO DE CONFIGURAÇÃO: Padrão regex inválido.\nDetalhes: {error}" } }, + "template": { + "name": "Entrada/Saída (I/O)", + "description": "Um modelo para avaliar trabalhos com base na entrada e saída de linha de comando." + } + }, + "static_analysis": { + "forbidden_import": { + "description": "Analisa estaticamente os arquivos de submissão para verificar se alguma biblioteca proibida foi importada. Penaliza a submissão caso alguma importação proibida seja encontrada.", + "params": { + "libraries": "Lista de nomes de bibliotecas/módulos cuja importação é proibida.", + "language": "Linguagem da submissão (ex: 'python', 'java'). Necessária para identificar padrões de importação corretos." + }, + "report": { + "no_imports": "Nenhuma importação proibida especificada — nada para verificar.", + "no_lang": "FALHA: A linguagem da submissão não foi definida. Não é possível verificar importações proibidas.", + "no_files": "Nenhum arquivo de submissão para analisar.", + "violation": " - '{lib}' encontrado em '{file}'", + "failure": "FALHA: Importações proibidas detectadas:\n{details}", + "success": "Nenhuma importação proibida encontrada." + } + }, "forbidden_keyword": { "description": "Analisa estaticamente os arquivos de submissão usando análise estrutural para detectar construções de linguagem proibidas (como loops ou chamadas eval).", "params": { @@ -503,6 +505,10 @@ "failure": "FALHA: Construções de linguagem proibidas detectadas:\n{details}", "success": "Nenhuma construção de linguagem proibida encontrada." } + }, + "template": { + "name": "Análise Estática", + "description": "Um modelo para avaliar tarefas através de análise estática e estrutural de código." } }, "api_testing": { @@ -542,4 +548,4 @@ "step_interrupted": "Ocorreu um erro inesperado durante a execução da etapa: {error}" } } -} \ No newline at end of file +} diff --git a/tests/unit/pipeline/test_load_template_step.py b/tests/unit/pipeline/test_load_template_step.py new file mode 100644 index 0000000..fe2405f --- /dev/null +++ b/tests/unit/pipeline/test_load_template_step.py @@ -0,0 +1,33 @@ +from unittest.mock import Mock, patch + +import pytest + +from autograder.steps.load_template_step import TemplateLoaderStep + + +@patch("autograder.steps.load_template_step.TemplateLibraryService.get_instance") +def test_normalizes_comma_separated_template_names(mock_get_instance): + mock_get_instance.return_value = Mock() + step = TemplateLoaderStep("input_output, static_analysis,") + assert step._template_names == ["input_output", "static_analysis"] + + +@patch("autograder.steps.load_template_step.TemplateLibraryService.get_instance") +def test_normalizes_template_name_list(mock_get_instance): + mock_get_instance.return_value = Mock() + step = TemplateLoaderStep(["input_output", " static_analysis ", ""]) + assert step._template_names == ["input_output", "static_analysis"] + + +@patch("autograder.steps.load_template_step.TemplateLibraryService.get_instance") +def test_rejects_empty_template_names_after_normalization(mock_get_instance): + mock_get_instance.return_value = Mock() + with pytest.raises(ValueError, match="at least one non-empty template identifier"): + TemplateLoaderStep(" , , ") + + +@patch("autograder.steps.load_template_step.TemplateLibraryService.get_instance") +def test_custom_template_allows_missing_template_names(mock_get_instance): + mock_get_instance.return_value = Mock() + step = TemplateLoaderStep(None, custom_template={"inline": "template"}) + assert step._template_names == [] diff --git a/tests/unit/pipeline/test_structural_analysis_step.py b/tests/unit/pipeline/test_structural_analysis_step.py index de3d85d..033271e 100644 --- a/tests/unit/pipeline/test_structural_analysis_step.py +++ b/tests/unit/pipeline/test_structural_analysis_step.py @@ -37,6 +37,7 @@ def test_structural_analysis_step_execution_success(mock_sg_root, mock_pipeline_ # Verify StepResult step_result = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS) assert step_result.status == StepStatus.SUCCESS + assert step_result.data.available is True assert "main.py" in step_result.data.roots assert "data.txt" not in step_result.data.roots # Heuristic should skip .txt assert step_result.data.roots["main.py"] == mock_root_instance @@ -55,6 +56,7 @@ def test_structural_analysis_step_parsing_failure(mock_sg_root, mock_pipeline_ex # Verify StepResult is still SUCCESS (graceful handling) step_result = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS) assert step_result.status == StepStatus.SUCCESS + assert step_result.data.available is True assert "main.py" in step_result.data.roots assert step_result.data.roots["main.py"] is None # Failed parsing results in None @@ -66,3 +68,15 @@ def test_structural_analysis_step_no_language(mock_pipeline_exec): step_result = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS) assert step_result.status == StepStatus.SUCCESS assert step_result.data.roots == {} + assert step_result.data.available is False + assert step_result.data.reason == "missing_submission_language" + +@patch("autograder.steps.structural_analysis_step.SgRoot", None) +def test_structural_analysis_step_unavailable_ast_grep(mock_pipeline_exec): + step = StructuralAnalysisStep() + result_exec = step.execute(mock_pipeline_exec) + + step_result = result_exec.get_step_result(StepName.STRUCTURAL_ANALYSIS) + assert step_result.status == StepStatus.SUCCESS + assert step_result.data.available is False + assert step_result.data.reason == "ast_grep_unavailable" diff --git a/tests/unit/test_command_resolution_at_build_time.py b/tests/unit/test_command_resolution_at_build_time.py index 71b0230..878ab40 100644 --- a/tests/unit/test_command_resolution_at_build_time.py +++ b/tests/unit/test_command_resolution_at_build_time.py @@ -7,13 +7,17 @@ from typing import List +from autograder.models.abstract.template import Template from autograder.services.grader.criteria_grader import SubmissionGrader from autograder.services.command_resolver import CommandResolver +from autograder.services.criteria_tree_service import CriteriaTreeService +from autograder.services.grader.grader_service import GraderService +from autograder.models.config.criteria import CriteriaConfig from autograder.models.criteria_tree import TestNode from autograder.models.abstract.test_function import TestFunction from autograder.models.dataclass.param_description import ParamDescription from autograder.models.dataclass.test_result import TestResult -from autograder.template_library.input_output import ForbiddenImportTest +from autograder.template_library.static_analysis import ForbiddenImportTest, StaticAnalysisTemplate from autograder.models.dataclass.submission import SubmissionFile from sandbox_manager.models.sandbox_models import Language @@ -45,6 +49,25 @@ def execute(self, files, sandbox, *args, **kwargs): return TestResult(test_name=self.name, score=100.0, report="ok") +class EmptyTemplate(Template): + """Template that never resolves tests, used to exercise multi-template lookup.""" + + @property + def template_name(self): + return "empty" + + @property + def template_description(self): + return "No tests" + + @property + def requires_sandbox(self): + return False + + def get_test(self, name): + raise KeyError(name) + + def _make_grader(language=None) -> SubmissionGrader: return SubmissionGrader( submission_files={}, @@ -303,3 +326,56 @@ def test_hidden_kwarg_is_not_accepted(self): ) # Language will be None → score 0 because it can't determine scan patterns assert result.score == 0.0 + + +class TestSubmissionLanguageCollisionRegression: + """Regression coverage for duplicate submission_language kwargs.""" + + def test_runtime_submission_language_overrides_config_value(self): + svc = _make_grader(Language.PYTHON) + fn = RecordingTestFunction() + node = _make_test_node({"submission_language": "java"}, fn) + + svc.process_test(node) + + assert fn.recorded_kwargs["submission_language"] == Language.PYTHON + + def test_config_submission_language_used_when_runtime_missing(self): + svc = _make_grader() + fn = RecordingTestFunction() + node = _make_test_node({"submission_language": "java"}, fn) + + svc.process_test(node) + + assert fn.recorded_kwargs["submission_language"] == "java" + + def test_multi_template_grading_path_with_submission_language_param(self): + criteria_dict = { + "base": { + "weight": 100, + "tests": [ + { + "name": "No forbidden imports", + "type": "forbidden_import", + "file": "main.py", + "parameters": [ + {"name": "forbidden_imports", "value": ["os"]}, + {"name": "submission_language", "value": "java"}, + ], + } + ], + } + } + criteria_config = CriteriaConfig.from_dict(criteria_dict) + criteria_tree = CriteriaTreeService().build_tree( + criteria_config, + [EmptyTemplate(), StaticAnalysisTemplate()], + ) + + result_tree = GraderService().grade_from_tree( + criteria_tree=criteria_tree, + submission_files={"main.py": SubmissionFile("main.py", "import os\n")}, + submission_language=Language.PYTHON, + ) + + assert result_tree.calculate_final_score() == 0.0 diff --git a/tests/unit/test_forbidden_import.py b/tests/unit/test_forbidden_import.py index 18a0928..89946ab 100644 --- a/tests/unit/test_forbidden_import.py +++ b/tests/unit/test_forbidden_import.py @@ -1,6 +1,6 @@ """Tests for ForbiddenImportTest.""" -from autograder.template_library.input_output import ForbiddenImportTest, InputOutputTemplate +from autograder.template_library.static_analysis import ForbiddenImportTest, StaticAnalysisTemplate from autograder.models.dataclass.submission import SubmissionFile from sandbox_manager.models.sandbox_models import Language @@ -9,8 +9,8 @@ class TestForbiddenImportRegistration: """Test that ForbiddenImportTest is properly registered in the template.""" def test_forbidden_import_registered_in_template(self): - """Test that the forbidden_import test is available in InputOutputTemplate.""" - template = InputOutputTemplate() + """Test that the forbidden_import test is available in StaticAnalysisTemplate.""" + template = StaticAnalysisTemplate() test = template.get_test("forbidden_import") assert test is not None assert test.name == "forbidden_import" @@ -473,6 +473,19 @@ def test_partial_module_name_no_false_positive(self): ) assert result.score == 100.0 + def test_special_regex_chars_in_module_names(self): + """Forbidden module names containing regex chars should still be matched literally.""" + files = [SubmissionFile("index.js", "require('foo.bar');\nrequire('a+b');\n")] + result = self.test_fn.execute( + files, + None, + forbidden_imports=["foo.bar", "a+b"], + submission_language=self.lang, + ) + assert result.score == 0.0 + assert "foo.bar" in result.report + assert "a+b" in result.report + def test_commented_require_still_detected(self): """Test that a commented require is still detected (known limitation). diff --git a/tests/unit/test_forbidden_keyword.py b/tests/unit/test_forbidden_keyword.py index 94d4ff2..aa9c1c8 100644 --- a/tests/unit/test_forbidden_keyword.py +++ b/tests/unit/test_forbidden_keyword.py @@ -4,7 +4,7 @@ import pytest from pydantic import ValidationError -from autograder.template_library.input_output import ForbiddenKeywordTest, InputOutputTemplate, ForbiddenKeywordConfig +from autograder.template_library.static_analysis import ForbiddenKeywordTest, StaticAnalysisTemplate, ForbiddenKeywordConfig from autograder.models.dataclass.submission import SubmissionFile from autograder.models.dataclass.structural_analysis_result import StructuralAnalysisResult from sandbox_manager.models.sandbox_models import Language @@ -14,8 +14,8 @@ class TestForbiddenKeywordRegistration: """Test that ForbiddenKeywordTest is properly registered in the template.""" def test_forbidden_keyword_registered_in_template(self): - """Test that the forbidden_keyword test is available in InputOutputTemplate.""" - template = InputOutputTemplate() + """Test that the forbidden_keyword test is available in StaticAnalysisTemplate.""" + template = StaticAnalysisTemplate() test = template.get_test("forbidden_keyword") assert test is not None assert test.name == "forbidden_keyword" @@ -85,6 +85,23 @@ def test_no_analysis_gives_0(self): assert result.score == 0.0 assert "analysis" in result.report.lower() + def test_unavailable_analysis_gives_0(self): + """Unavailable structural analysis should fail explicitly.""" + sa_result = StructuralAnalysisResult( + roots={}, + available=False, + reason="ast_grep_unavailable", + ) + result = self.test_fn.execute( + [SubmissionFile("main.py", "for i in range(10): pass")], + None, + forbidden_keywords=["for_loop"], + structural_analysis=sa_result, + submission_language=Language.PYTHON, + ) + assert result.score == 0.0 + assert "analysis" in result.report.lower() + def test_no_language_gives_0(self): """Test that missing language returns score 0.""" sa_result = StructuralAnalysisResult(roots={}) @@ -153,3 +170,17 @@ def test_no_violation_found(self): submission_language=Language.PYTHON) assert result.score == 100.0 + + def test_missing_roots_for_target_file_gives_0(self): + """If target files have no parsed roots, the test should fail as no-analysis.""" + sa_result = StructuralAnalysisResult(roots={}) + files = [SubmissionFile("main.py", "for i in range(10): pass")] + result = self.test_fn.execute( + files, + None, + forbidden_keywords=["for_loop"], + structural_analysis=sa_result, + submission_language=Language.PYTHON, + ) + assert result.score == 0.0 + assert "analysis" in result.report.lower() diff --git a/tests/unit/test_template_library_service.py b/tests/unit/test_template_library_service.py new file mode 100644 index 0000000..d986d44 --- /dev/null +++ b/tests/unit/test_template_library_service.py @@ -0,0 +1,12 @@ +from autograder.services.template_library_service import TemplateLibraryService + + +def test_template_info_exposes_stable_identifier_and_display_name(): + TemplateLibraryService.reset_instance() + service = TemplateLibraryService.get_instance() + + info = service.get_template_info("static_analysis") + + assert info["identifier"] == "static_analysis" + assert isinstance(info["name"], str) + assert info["name"] diff --git a/tests/unit/test_test_function_validation.py b/tests/unit/test_test_function_validation.py index b38ce80..e3b4812 100644 --- a/tests/unit/test_test_function_validation.py +++ b/tests/unit/test_test_function_validation.py @@ -71,7 +71,7 @@ def test_validation_success(self): } config = CriteriaConfig.from_dict(criteria_dict) # Should NOT raise ValueError - tree = self.service.build_tree(config, self.template) + tree = self.service.build_tree(config, [self.template]) assert tree is not None assert tree.base.tests[0].parameters["required_str"] == "hello" @@ -93,7 +93,7 @@ def test_validation_failure_missing_field(self): } config = CriteriaConfig.from_dict(criteria_dict) with pytest.raises(ValueError) as excinfo: - self.service.build_tree(config, self.template) + self.service.build_tree(config, [self.template]) assert "Invalid parameters" in str(excinfo.value) assert "required_str" in str(excinfo.value) @@ -117,7 +117,7 @@ def test_validation_failure_wrong_type(self): } config = CriteriaConfig.from_dict(criteria_dict) with pytest.raises(ValueError) as excinfo: - self.service.build_tree(config, self.template) + self.service.build_tree(config, [self.template]) assert "Invalid parameters" in str(excinfo.value) assert "optional_int" in str(excinfo.value)