diff --git a/.github/scripts/cldc11_api_compat_check.py b/.github/scripts/cldc11_api_compat_check.py new file mode 100644 index 0000000000..02e17d5f88 --- /dev/null +++ b/.github/scripts/cldc11_api_compat_check.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +""" +Validate that the CLDC11 API surface is a binary compatible subset of Java SE 11 +and the vm/JavaAPI project. + +The script compares public and protected methods and fields using `javap -s -public`. +It expects pre-built class directories for the CLDC11 and JavaAPI projects. +""" +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from typing import Dict, Iterable, List, Optional, Set, Tuple + + +def log(message: str) -> None: + """Emit a progress message immediately.""" + + print(message, flush=True) + + +Member = Tuple[str, str, bool, str] +""" +A member descriptor in the form `(name, descriptor, is_static, kind)`. +`kind` is either `method` or `field`. +""" + + +@dataclass +class ApiSurface: + """Collection of public/protected API members for a class.""" + + methods: Set[Member] = field(default_factory=set) + fields: Set[Member] = field(default_factory=set) + + def add(self, member: Member) -> None: + if member[3] == "method": + self.methods.add(member) + else: + self.fields.add(member) + + def missing_from(self, other: "ApiSurface") -> Tuple[Set[Member], Set[Member]]: + return self.methods - other.methods, self.fields - other.fields + + def extras_over(self, other: "ApiSurface") -> Tuple[Set[Member], Set[Member]]: + return self.methods - other.methods, self.fields - other.fields + + +class JavapError(RuntimeError): + pass + + +@dataclass +class ClassInfo: + """Metadata about a compiled class.""" + + name: str + api: ApiSurface + supers: List[str] + is_public: bool + kind: str + + +def discover_classes(root: str) -> List[str]: + classes: List[str] = [] + for base, _, files in os.walk(root): + for filename in files: + if not filename.endswith(".class"): + continue + if filename in {"module-info.class", "package-info.class"}: + continue + full_path = os.path.join(base, filename) + rel_path = os.path.relpath(full_path, root) + binary_name = rel_path[:-6].replace(os.sep, ".") + classes.append(binary_name) + return classes + + +def run_javap(target: str, javap_cmd: str) -> str: + proc = subprocess.run( + [javap_cmd, "-public", "-s", target], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if proc.returncode != 0: + raise JavapError(proc.stderr.strip() or proc.stdout.strip()) + return proc.stdout + + +def parse_members(javap_output: str) -> ApiSurface: + api = ApiSurface() + pending: Optional[Tuple[str, bool, str]] = None # name, is_static, kind + + for raw_line in javap_output.splitlines(): + line = raw_line.strip() + if not line or line.startswith("Compiled from"): + continue + if line.endswith("{"): + continue + if line.startswith("descriptor:"): + if pending is None: + continue + descriptor = line.split(":", 1)[1].strip() + name, is_static, kind = pending + api.add((name, descriptor, is_static, kind)) + pending = None + continue + + if line.startswith("Runtime") or line.startswith("Signature:") or line.startswith("Exceptions:"): + pending = None + continue + + if "(" in line or line.endswith(";"): + if line.startswith("//"): + continue + if line.endswith(" class"): + continue + if line.endswith("interface"): + continue + + is_static = " static " in f" {line} " + if "(" in line: + name_section = line.split("(")[0].strip() + name = name_section.split()[-1] + kind = "method" + else: + name = line.rstrip(";").split()[-1] + kind = "field" + pending = (name, is_static, kind) + + return api + + +def parse_class_info(javap_output: str) -> ClassInfo: + api = ApiSurface() + supers: List[str] = [] + pending: Optional[Tuple[str, bool, str]] = None + class_name: Optional[str] = None + is_public = False + kind = "class" + + header_pattern = re.compile( + r"(?Ppublic|protected)?\s*(?Pclass|interface|enum)\s+" + r"(?P[\w.$]+)" + r"(?:\s+extends\s+(?P[^\{]+?)(?=\s+implements|$))?" + r"(?:\s+implements\s+(?P[^\{]+))?", + ) + + for raw_line in javap_output.splitlines(): + line = raw_line.strip() + if not line or line.startswith("Compiled from"): + continue + + if class_name is None: + match = header_pattern.search(line.rstrip("{")) + if match: + class_name = match.group("name") + kind = match.group("kind") + is_public = match.group("visibility") == "public" + extends_clause = match.group("extends") + implements_clause = match.group("implements") + for clause in (extends_clause, implements_clause): + if not clause: + continue + supers.extend([part.strip() for part in clause.split(',') if part.strip()]) + continue + + if line.endswith("{"): + continue + if line.startswith("descriptor:"): + if pending is None: + continue + descriptor = line.split(":", 1)[1].strip() + name, is_static, kind = pending + api.add((name, descriptor, is_static, kind)) + pending = None + continue + + if line.startswith("Runtime") or line.startswith("Signature:") or line.startswith("Exceptions:"): + pending = None + continue + + if "(" in line or line.endswith(";"): + if line.startswith("//"): + continue + if line.endswith(" class"): + continue + if line.endswith("interface"): + continue + + is_static_member = " static " in f" {line} " + if "(" in line: + name_section = line.split("(")[0].strip() + name = name_section.split()[-1] + kind = "method" + else: + name = line.rstrip(";").split()[-1] + kind = "field" + pending = (name, is_static_member, kind) + + if class_name is None: + raise ValueError("Unable to determine class name from javap output") + + if not supers and kind == "class" and class_name != "java.lang.Object": + supers.append("java.lang.Object") + + return ClassInfo(name=class_name, api=api, supers=supers, is_public=is_public, kind=kind) + +def collect_class_info_from_file(class_name: str, classes_root: str, javap_cmd: str) -> Optional[ClassInfo]: + class_path = os.path.join(classes_root, *class_name.split(".")) + ".class" + if not os.path.exists(class_path): + return None + output = run_javap(class_path, javap_cmd) + return parse_class_info(output) + + +def collect_class_info_from_jdk(class_name: str, javap_cmd: str) -> Optional[ClassInfo]: + try: + output = run_javap(class_name, javap_cmd) + except JavapError: + return None + return parse_class_info(output) + + +def format_member(member: Member) -> str: + name, descriptor, is_static, kind = member + static_prefix = "static " if is_static else "" + return f"{kind}: {static_prefix}{name} {descriptor}" + + +def build_full_api( + class_name: str, + lookup, + cache: Dict[str, Optional[ApiSurface]], +) -> Optional[ApiSurface]: + if class_name in cache: + return cache[class_name] + + info = lookup(class_name) + if info is None: + cache[class_name] = None + return None + + merged = ApiSurface(set(info.api.methods), set(info.api.fields)) + for parent in info.supers: + parent_api = build_full_api(parent, lookup, cache) + if parent_api: + merged.methods |= parent_api.methods + merged.fields |= parent_api.fields + + cache[class_name] = merged + return merged + + +def ensure_subset( + source_classes: List[str], + source_lookup, + target_lookup, + target_label: str, +) -> Tuple[bool, List[str]]: + ok = True + messages: List[str] = [] + source_cache: Dict[str, Optional[ApiSurface]] = {} + target_cache: Dict[str, Optional[ApiSurface]] = {} + + for index, class_name in enumerate(sorted(source_classes), start=1): + source_info = source_lookup(class_name) + if source_info is None: + ok = False + messages.append(f"Failed to read {class_name} from source classes") + continue + + source_api = build_full_api(class_name, source_lookup, source_cache) + assert source_api is not None + + if index % 25 == 0: + log(f" Processed {index}/{len(source_classes)} classes for {target_label} subset check...") + + target_info = target_lookup(class_name) + if target_info is None: + ok = False + messages.append(f"Missing class in {target_label}: {class_name}") + continue + + target_api = build_full_api(class_name, target_lookup, target_cache) + if target_api is None: + ok = False + messages.append(f"Failed to read {class_name} from {target_label}") + continue + + extra_supers = [stype for stype in source_info.supers if stype not in target_info.supers] + if extra_supers: + ok = False + messages.append(f"Incompatibilities for {class_name} against {target_label}:") + for stype in extra_supers: + messages.append(f" - supertype: {stype} not present in {target_label} declaration") + continue + + missing_methods, missing_fields = source_api.missing_from(target_api) + if missing_methods or missing_fields: + ok = False + messages.append(f"Incompatibilities for {class_name} against {target_label}:") + for member in sorted(missing_methods | missing_fields): + messages.append(f" - {format_member(member)}") + + return ok, messages + + +def collect_javaapi_map(javaapi_root: str, javap_cmd: str) -> Dict[str, ClassInfo]: + classes = discover_classes(javaapi_root) + api_map: Dict[str, ClassInfo] = {} + for index, class_name in enumerate(classes, start=1): + info = collect_class_info_from_file(class_name, javaapi_root, javap_cmd) + if info: + api_map[class_name] = info + if index % 25 == 0: + log(f" Indexed {index}/{len(classes)} vm/JavaAPI classes...") + return api_map + + +def write_extra_report( + cldc_classes: Dict[str, ClassInfo], + javaapi_classes: Dict[str, ClassInfo], + public_cldc: Set[str], + report_path: str, +) -> None: + lines: List[str] = [ + "Extra APIs present in vm/JavaAPI but not in CLDC11", + "", + ] + + extra_classes = sorted( + name + for name, info in javaapi_classes.items() + if info.is_public and name not in public_cldc + ) + if extra_classes: + lines.append("Classes only in vm/JavaAPI:") + lines.extend([f" - {name}" for name in extra_classes]) + lines.append("") + + shared_classes = {name for name in javaapi_classes if name in public_cldc and javaapi_classes[name].is_public} + extra_members: List[str] = [] + for class_name in sorted(shared_classes): + javaapi_api = javaapi_classes[class_name].api + cldc_api = cldc_classes[class_name].api + extra_methods, extra_fields = javaapi_api.extras_over(cldc_api) + if not extra_methods and not extra_fields: + continue + extra_members.append(class_name) + extra_members.append("") + for member in sorted(extra_methods | extra_fields): + extra_members.append(f" + {format_member(member)}") + extra_members.append("") + + if extra_members: + lines.append("Additional members on classes shared between vm/JavaAPI and CLDC11:") + lines.append("") + lines.extend(extra_members) + + if len(lines) == 2: + lines.append("No extra APIs detected.") + + os.makedirs(os.path.dirname(os.path.abspath(report_path)) or ".", exist_ok=True) + with open(report_path, "w", encoding="utf-8") as handle: + handle.write("\n".join(lines).rstrip() + "\n") + + +def main(argv: Optional[Iterable[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cldc-classes", required=True, help="Path to compiled CLDC11 classes directory") + parser.add_argument("--javaapi-classes", required=True, help="Path to compiled vm/JavaAPI classes directory") + parser.add_argument("--extra-report", required=True, help="File path to write the extra API report") + parser.add_argument( + "--javap", + default=( + os.path.join(os.environ.get("JAVA_HOME", ""), "bin", "javap") + if os.environ.get("JAVA_HOME") + else "javap" + ), + help="Path to the javap executable from Java SE 11", + ) + args = parser.parse_args(argv) + + javap_cmd = args.javap or "javap" + + cldc_class_names = discover_classes(args.cldc_classes) + if not cldc_class_names: + print(f"No class files found under {args.cldc_classes}", file=sys.stderr) + return 1 + + log(f"Discovered {len(cldc_class_names)} CLDC11 classes; building API maps...") + + javaapi_map = collect_javaapi_map(args.javaapi_classes, javap_cmd) + log(f"Collected API surface for {len(javaapi_map)} vm/JavaAPI classes") + + cldc_lookup_cache: Dict[str, Optional[ClassInfo]] = {} + java_lookup_cache: Dict[str, Optional[ClassInfo]] = {name: info for name, info in javaapi_map.items()} + jdk_lookup_cache: Dict[str, Optional[ClassInfo]] = {} + + def cldc_lookup(name: str) -> Optional[ClassInfo]: + if name not in cldc_lookup_cache: + cldc_lookup_cache[name] = collect_class_info_from_file(name, args.cldc_classes, javap_cmd) + return cldc_lookup_cache[name] + + def jdk_lookup(name: str) -> Optional[ClassInfo]: + if name not in jdk_lookup_cache: + jdk_lookup_cache[name] = collect_class_info_from_jdk(name, javap_cmd) + return jdk_lookup_cache[name] + + def javaapi_lookup(name: str) -> Optional[ClassInfo]: + return java_lookup_cache.get(name) + + public_cldc_classes = [ + name for name in cldc_class_names if (cldc_lookup(name) and cldc_lookup_cache[name].is_public) + ] + + if not public_cldc_classes: + print("No public classes discovered in CLDC11 output", file=sys.stderr) + return 1 + + log("Validating CLDC11 API against Java SE 11...") + java_ok, java_messages = ensure_subset( + public_cldc_classes, + cldc_lookup, + jdk_lookup, + "Java SE 11", + ) + + log("Validating CLDC11 API against vm/JavaAPI...") + api_ok, api_messages = ensure_subset( + public_cldc_classes, + cldc_lookup, + javaapi_lookup, + "vm/JavaAPI", + ) + + cldc_map = {name: info for name, info in cldc_lookup_cache.items() if info is not None} + write_extra_report(cldc_map, javaapi_map, set(public_cldc_classes), args.extra_report) + log(f"Wrote extra API report to {args.extra_report}") + + messages = java_messages + api_messages + if messages: + print("\n".join(messages), file=sys.stderr) + return 0 if java_ok and api_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/cldc11-api-compatibility.yml b/.github/workflows/cldc11-api-compatibility.yml new file mode 100644 index 0000000000..5b263a0f9e --- /dev/null +++ b/.github/workflows/cldc11-api-compatibility.yml @@ -0,0 +1,62 @@ +name: CLDC11 API Compatibility + +on: + push: + branches: [master] + paths: + - 'Ports/CLDC11/**' + - 'vm/JavaAPI/**' + - '.github/workflows/cldc11-api-compatibility.yml' + - '.github/scripts/cldc11_api_compat_check.py' + pull_request: + paths: + - 'Ports/CLDC11/**' + - 'vm/JavaAPI/**' + - '.github/workflows/cldc11-api-compatibility.yml' + - '.github/scripts/cldc11_api_compat_check.py' + +jobs: + api-compatibility: + runs-on: ubuntu-latest + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up JDK 8 for builds + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '8' + + - name: Compile CLDC11 classes + run: | + mkdir -p Ports/CLDC11/build/classes + find Ports/CLDC11/src/java -name '*.java' > cldc11-sources.txt + javac -encoding windows-1252 -source 1.6 -target 1.6 -XDignore.symbol.file \ + -d Ports/CLDC11/build/classes @cldc11-sources.txt + + - name: Build vm/JavaAPI classes + run: mvn -f vm/pom.xml -pl JavaAPI -am package -DskipTests + + - name: Set up JDK 11 for compatibility checks + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '11' + + - name: Validate CLDC11 API subset + run: | + python .github/scripts/cldc11_api_compat_check.py \ + --cldc-classes Ports/CLDC11/build/classes \ + --javaapi-classes vm/JavaAPI/target/classes \ + --extra-report cldc11-extra-apis.txt + + - name: Upload extra API report + if: always() + uses: actions/upload-artifact@v4 + with: + name: cldc11-extra-apis + path: cldc11-extra-apis.txt