diff --git a/fcli-core/fcli-aviator-common/build.gradle.kts b/fcli-core/fcli-aviator-common/build.gradle.kts index ee6efdf5437..f2656bb3e7b 100644 --- a/fcli-core/fcli-aviator-common/build.gradle.kts +++ b/fcli-core/fcli-aviator-common/build.gradle.kts @@ -14,10 +14,15 @@ tasks.withType().configureEach { dependsOn("generateProto") } dependencies { implementation(project(":fcli-core:fcli-common")) implementation("org.yaml:snakeyaml:2.3") - implementation("jakarta.xml.bind:jakarta.xml.bind-api:2.3.3") - implementation("org.glassfish.jaxb:jaxb-runtime:2.3.3") + + // JAXB for XML object marshalling (used in FVDLProcessor legacy parser) + implementation("jakarta.xml.bind:jakarta.xml.bind-api:3.0.1") + implementation("org.glassfish.jaxb:jaxb-runtime:3.0.2") implementation("com.sun.activation:jakarta.activation:2.0.1") - implementation("jakarta.xml.ws:jakarta.xml.ws-api:3.0.1") + + // Note: StAX (javax.xml.stream) uses Woodstox 7.1.1 via jackson-dataformat-xml + // from fcli-common (needed for XML output). No explicit dependency required. + implementation("com.auth0:java-jwt:4.5.0") implementation("io.grpc:grpc-netty-shaded:1.76.0") implementation("io.grpc:grpc-protobuf:1.76.0") diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java index 4abe82d1914..f817251971a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java @@ -37,7 +37,7 @@ import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; -import com.fortify.cli.aviator.fpr.processor.FVDLProcessor; +import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.ResourceUtil; @@ -65,13 +65,13 @@ public static FPRAuditResult auditFPR(AuditFprOptions options) Map auditResponses = new ConcurrentHashMap<>(); AuditOutcome auditOutcome = performAviatorAudit( parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(), - auditResponses, filterSelection + auditResponses, filterSelection, options.getFprHandle() ); // --- STAGE 4: FINALIZATION --- return finalizeFprAudit( auditOutcome, auditResponses, parsedData.auditProcessor, - tagMappingConfig, parsedData.fprInfo, parsedData.fvdlProcessor + tagMappingConfig, parsedData.fprInfo ); } @@ -79,14 +79,15 @@ private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle) { try { // Processors now take the FprHandle directly, no more extracted path AuditProcessor auditProcessor = new AuditProcessor(fprHandle); - FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); + //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle); + StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle); Map auditIssueMap = auditProcessor.processAuditXML(); FPRProcessor fprProcessor = new FPRProcessor(fprHandle, auditIssueMap, auditProcessor); - List vulnerabilities = fprProcessor.process(fvdlProcessor); + List vulnerabilities = fprProcessor.process(streamingFVDLProcessor); FPRInfo fprInfo = fprProcessor.getFprInfo(); - return new ParsedFprData(auditIssueMap, vulnerabilities, fprInfo, auditProcessor, fvdlProcessor); + return new ParsedFprData(auditIssueMap, vulnerabilities, fprInfo, auditProcessor, streamingFVDLProcessor); } catch (Exception e) { LOG.error("A critical error occurred during FPR processing.", e); throw new AviatorTechnicalException("Failed to process FPR contents.", e); @@ -106,21 +107,21 @@ private static TagMappingConfig loadTagMappingConfig(String tagMappingFilePath) private static AuditOutcome performAviatorAudit( ParsedFprData parsedData, IAviatorLogger logger, String token, String appVersion, String url, String sscAppName, String sscAppVersion, - Map auditResponsesToFill, FilterSelection filterSelection) { + Map auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle) { IssueAuditor issueAuditor = new IssueAuditor( parsedData.vulnerabilities, parsedData.auditProcessor, parsedData.auditIssueMap, parsedData.fprInfo, sscAppName, sscAppVersion, filterSelection, logger ); return issueAuditor.performAudit( - auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url + auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle ); } private static FPRAuditResult finalizeFprAudit( AuditOutcome auditOutcome, Map auditResponses, AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig, - FPRInfo fprInfo, FVDLProcessor fvdlProcessor) { + FPRInfo fprInfo) { int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit(); if (auditResponses.isEmpty()) { @@ -160,10 +161,10 @@ private static FPRAuditResult finalizeFprAudit( File updatedFile = null; if (issuesSuccessfullyAudited > 0) { - updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, fprInfo, fvdlProcessor); + updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, fprInfo); } LOG.info("FPR audit process completed with status: {}", status); return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit); } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java index e67741be27c..703a886afa4 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java @@ -51,6 +51,7 @@ import com.fortify.cli.aviator.grpc.AviatorGrpcClient; import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper; import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FprHandle; import com.fortify.cli.aviator.util.StringUtil; @@ -141,7 +142,7 @@ private TagDefinition resolveHumanAuditStatus() { } public AuditOutcome performAudit(Map auditResponses, String token, - String projectName, String projectBuildId, String url) { + String projectName, String projectBuildId, String url, FprHandle fprHandle) { projectName = StringUtil.isEmpty(projectName) ? projectBuildId : projectName; logger.progress("Starting audit for project: %s", projectName); @@ -154,7 +155,7 @@ public AuditOutcome performAudit(Map auditResponses, Stri } else { try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) { CompletableFuture> future = - client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token); + client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token, fprHandle); Map responses = future.get(500, TimeUnit.MINUTES); responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response)); logger.progress("Audit completed"); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/ParsedFprData.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/ParsedFprData.java index b7fcf1d67c3..9f70dd3f2f9 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/ParsedFprData.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/ParsedFprData.java @@ -19,7 +19,7 @@ import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; -import com.fortify.cli.aviator.fpr.processor.FVDLProcessor; +import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; /** * A data-holding class that represents the complete, parsed contents of an FPR file. @@ -31,13 +31,14 @@ public final class ParsedFprData { public final List vulnerabilities; public final FPRInfo fprInfo; public final AuditProcessor auditProcessor; - public final FVDLProcessor fvdlProcessor; + //public final FVDLProcessor fvdlProcessor; + public final StreamingFVDLProcessor streamingFVDLProcessor; - public ParsedFprData(Map auditIssueMap, List vulnerabilities, FPRInfo fprInfo, AuditProcessor auditProcessor, FVDLProcessor fvdlProcessor) { + public ParsedFprData(Map auditIssueMap, List vulnerabilities, FPRInfo fprInfo, AuditProcessor auditProcessor, StreamingFVDLProcessor streamingFvdlProcessor) { this.auditIssueMap = auditIssueMap; this.vulnerabilities = vulnerabilities; this.fprInfo = fprInfo; this.auditProcessor = auditProcessor; - this.fvdlProcessor = fvdlProcessor; + this.streamingFVDLProcessor = streamingFvdlProcessor; } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/config/LanguagesCommentConfig.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/config/LanguagesCommentConfig.java new file mode 100644 index 00000000000..75538f793a0 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/config/LanguagesCommentConfig.java @@ -0,0 +1,38 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.config; + +import java.util.HashMap; +import java.util.Map; + +import com.fortify.cli.aviator.util.StringUtil; + +public class LanguagesCommentConfig { + private Map lineCommentSymbols = new HashMap<>(); + + public void setLineCommentSymbols(Map comments) { + this.lineCommentSymbols = comments != null ? new HashMap<>(comments) : new HashMap<>(); + } + + public Map getLineCommentSymbols() { + return new HashMap<>(lineCommentSymbols); + } + + public String getCommentForLanguage(String language) { + if (StringUtil.isEmpty(language)) { + return "Unknown"; + } + + return lineCommentSymbols.getOrDefault(language, "Unknown"); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java index 618f5aab7c8..0219fbed94a 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.zip.ZipFile; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,8 +26,8 @@ import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; import com.fortify.cli.aviator.fpr.processor.AuditProcessor; -import com.fortify.cli.aviator.fpr.processor.FVDLProcessor; import com.fortify.cli.aviator.fpr.processor.FilterTemplateParser; +import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor; import com.fortify.cli.aviator.util.FprHandle; import lombok.Getter; @@ -50,10 +51,10 @@ public FPRProcessor(FprHandle fprHandle, Map auditIssueMap, /** * Processes the main components of the FPR. * - * @param fvdlProcessor The processor for handling the audit.fvdl file. + * @param streamingFVDLProcessor The processor for handling the audit.fvdl file. * @return A list of all vulnerabilities found in the FVDL. */ - public List process(FVDLProcessor fvdlProcessor) { + public List process(StreamingFVDLProcessor streamingFVDLProcessor) { logger.info("FPR Processing started"); try{ this.fprInfo = new FPRInfo(this.fprHandle); @@ -83,7 +84,13 @@ public List process(FVDLProcessor fvdlProcessor) { logger.debug("Audit.xml Issues found: {}", auditIssueMap.size()); - List vulnerabilities = fvdlProcessor.processXML(); + try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) { + streamingFVDLProcessor.parse(zipFile, "audit.fvdl"); + } + + //List vulnerabilities = fvdlProcessor.processXML(); + + List vulnerabilities = streamingFVDLProcessor.getVulnerabilities(); logger.info("Parsed {} vulnerabilities from FVDL.", vulnerabilities.size()); return vulnerabilities; @@ -94,4 +101,4 @@ public List process(FVDLProcessor fvdlProcessor) { throw new AviatorTechnicalException("Unexpected error during FPR processing.", e); } } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FPRInfo.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FPRInfo.java index d7e0ed35fb8..650f5d92e19 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FPRInfo.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FPRInfo.java @@ -53,13 +53,127 @@ public class FPRInfo { public FPRInfo(FprHandle fprHandle) { FPRName = String.valueOf(fprHandle.getFprPath().getFileName()); try { - extractInfoFromAuditFvdl(fprHandle); + extractInfoFromAuditFvdlStreaming(fprHandle); } catch (Exception e) { // It's better to wrap this in a specific runtime exception throw new RuntimeException("Failed to extract info from audit.fvdl", e); } } + /** + * Extract FPR metadata from audit.fvdl using streaming XML parsing (StAX). + * More memory-efficient than DOM parsing for large files. + * + * Extracts: + * - UUID + * - Build information (BuildID, SourceBasePath, NumberFiles, ScanTime) + * + * @param fprHandle for getting path + * @throws Exception if parsing fails + */ + private void extractInfoFromAuditFvdlStreaming(FprHandle fprHandle) throws Exception { + Path auditPath = fprHandle.getPath("/audit.fvdl"); + + if (!Files.exists(auditPath)) { + throw new IllegalStateException("audit.fvdl not found in FPR: " + fprHandle.getFprPath()); + } + + javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newInstance(); + // Security: Disable external entity processing + factory.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + factory.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, false); + + try (java.io.InputStream inputStream = Files.newInputStream(auditPath)) { + javax.xml.stream.XMLStreamReader reader = factory.createXMLStreamReader(inputStream); + + boolean inBuild = false; + String currentElement = null; + + while (reader.hasNext()) { + int event = reader.next(); + + if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("UUID".equals(localName)) { + // Extract UUID text content + this.uuid = readElementText(reader); + + } else if ("Build".equals(localName)) { + // Entering Build section + inBuild = true; + + } else if (inBuild) { + // Inside Build section, capture element name + currentElement = localName; + } + + } else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS && inBuild && currentElement != null) { + // Read text content for Build child elements + String text = reader.getText().trim(); + if (!text.isEmpty()) { + switch (currentElement) { + case "BuildID": + this.buildId = text; + break; + case "SourceBasePath": + this.sourceBasePath = text; + break; + case "NumberFiles": + this.numberOfFiles = parseIntegerContent(text); + break; + case "ScanTime": + this.scanTime = parseIntegerContent(text); + break; + } + } + + } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Build".equals(localName)) { + // Exiting Build section, we have all needed data + inBuild = false; + // Early exit: we've extracted all needed metadata + if (this.uuid != null) { + break; // Stop parsing, we have everything + } + + } else if (inBuild) { + // Clear current element when exiting child element + currentElement = null; + } + } + } + + reader.close(); + + } catch (javax.xml.stream.XMLStreamException e) { + throw new Exception("Failed to parse audit.fvdl using streaming parser", e); + } + } + + /** + * Helper method to read element text content using StAX reader. + * Advances reader to the text content and returns it. + * + * @param reader XMLStreamReader positioned at START_ELEMENT + * @return Text content of the element, or empty string if no text + * @throws javax.xml.stream.XMLStreamException if reading fails + */ + private String readElementText(javax.xml.stream.XMLStreamReader reader) throws javax.xml.stream.XMLStreamException { + StringBuilder text = new StringBuilder(); + while (reader.hasNext()) { + int event = reader.next(); + if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) { + text.append(reader.getText()); + } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) { + break; + } + } + return text.toString().trim(); + } + private void extractInfoFromAuditFvdl(FprHandle fprHandle) throws Exception { Path auditPath = fprHandle.getPath("/audit.fvdl"); @@ -127,4 +241,4 @@ public Optional getDefaultEnabledFilterSet() { .filter(FilterSet::isEnabled) .findFirst(); } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FVDLMetadata.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FVDLMetadata.java new file mode 100644 index 00000000000..49e3b9f0a35 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FVDLMetadata.java @@ -0,0 +1,61 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.model; + + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import lombok.Data; + +/** + * Container for FPR metadata extracted during streaming parse. + */ +@Data +public class FVDLMetadata { + + private String buildId; + private String projectName; + private String projectVersion; + private String engineVersion; + + // Rule metadata cache: classId -> metadata map + private Map> ruleMetadata = new ConcurrentHashMap<>(); + + // Node pool: nodeId -> minimal node data + private Map nodePool = new ConcurrentHashMap<>(); + + // Trace pool: traceId -> StreamedTrace + private Map tracePool = new ConcurrentHashMap<>(); + + // Description cache: classID -> StreamedDescription + private Map descriptionCache = new ConcurrentHashMap<>(); + + // Statistics + private long totalVulnerabilities; + private long totalNodes; + private long totalTraces; + + + @Data + public static class NodeData { + private String nodeId; + private String filePath; + private Integer line; + private Integer lineEnd; + private Integer colStart; + private Integer colEnd; + private String actionType; + private String label; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/Node.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/Node.java index e34552d4386..c9cb6cdc660 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/Node.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/Node.java @@ -12,27 +12,44 @@ */ package com.fortify.cli.aviator.fpr.model; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +import com.fortify.cli.aviator.fpr.filter.FilterSet; +import com.fortify.cli.aviator.fpr.filter.FilterTemplate; import com.fortify.cli.aviator.fpr.utils.Searchable; +import com.fortify.cli.aviator.util.StringUtil; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; + /** * Represents a node in the FVDL UnifiedNodePool or trace entry. * Enhanced with taintFlags, knowledge, secondary location, detailsOnly, label, and reasonText for full coverage. * Implements Searchable for description conditionals. + * + * Note: Uses manual constructors instead of @AllArgsConstructor for constructor overloading + * to support both DOM parser (22 params) and Streaming parser (24 params) use cases. */ @Getter @Setter -@AllArgsConstructor public class Node implements Searchable { + // Existing fields private String id; private String filePath; private int line; @@ -45,9 +62,11 @@ public class Node implements Searchable { private String additionalInfo; private String associatedRuleId; + // Existing enhanced fields private List taintFlags = new ArrayList<>(); // From Knowledge/Fact type="TaintFlags" private Map knowledge = new HashMap<>(); // From Knowledge/Fact (type -> value) + // New fields private String secondaryPath = ""; // From SecondaryLocation private int secondaryLine = 0; private int secondaryLineEnd = 0; @@ -58,6 +77,12 @@ public class Node implements Searchable { private String label = ""; // From @label private String reasonText = ""; // From Reason/Internal or concatenated reason info + // InnerStackTrace support fields (Option B - Simplified approach) + // Used by Streaming parser to store Reason trace data for building innerStackTrace + // DOM parser doesn't use these (uses rawNodePool with JAXB objects instead) + private List reasonTraceRefs = new ArrayList<>(); // TraceRef IDs from Reason element + private List reasonInlineTraces = new ArrayList<>(); // Inline Traces from Reason element + /** * Checks if any searchable field contains the given string. * @@ -71,14 +96,14 @@ public boolean contains(String searchString) { } String lowerSearch = searchString.toLowerCase(); return (filePath != null && filePath.toLowerCase().contains(lowerSearch)) - || (secondaryPath != null && secondaryPath.toLowerCase().contains(lowerSearch)) - || (actionType != null && actionType.toLowerCase().contains(lowerSearch)) - || (additionalInfo != null && additionalInfo.toLowerCase().contains(lowerSearch)) - || (associatedRuleId != null && associatedRuleId.toLowerCase().contains(lowerSearch)) - || (reasonText != null && reasonText.toLowerCase().contains(lowerSearch)) - || (label != null && label.toLowerCase().contains(lowerSearch)) - || taintFlags.stream().anyMatch(flag -> flag.toLowerCase().contains(lowerSearch)) - || knowledge.values().stream().anyMatch(value -> value != null && value.toLowerCase().contains(lowerSearch)); + || (secondaryPath != null && secondaryPath.toLowerCase().contains(lowerSearch)) + || (actionType != null && actionType.toLowerCase().contains(lowerSearch)) + || (additionalInfo != null && additionalInfo.toLowerCase().contains(lowerSearch)) + || (associatedRuleId != null && associatedRuleId.toLowerCase().contains(lowerSearch)) + || (reasonText != null && reasonText.toLowerCase().contains(lowerSearch)) + || (label != null && label.toLowerCase().contains(lowerSearch)) + || taintFlags.stream().anyMatch(flag -> flag.toLowerCase().contains(lowerSearch)) + || knowledge.values().stream().anyMatch(value -> value != null && value.toLowerCase().contains(lowerSearch)); } /** @@ -94,14 +119,14 @@ public boolean matches(String matchString) { } String lowerMatch = matchString.toLowerCase(); return (filePath != null && filePath.toLowerCase().equals(lowerMatch)) - || (secondaryPath != null && secondaryPath.toLowerCase().equals(lowerMatch)) - || (actionType != null && actionType.toLowerCase().equals(lowerMatch)) - || (additionalInfo != null && additionalInfo.toLowerCase().equals(lowerMatch)) - || (associatedRuleId != null && associatedRuleId.toLowerCase().equals(lowerMatch)) - || (reasonText != null && reasonText.toLowerCase().equals(lowerMatch)) - || (label != null && label.toLowerCase().equals(lowerMatch)) - || taintFlags.stream().anyMatch(flag -> flag.toLowerCase().equals(lowerMatch)) - || knowledge.values().stream().anyMatch(value -> value != null && value.toLowerCase().equals(lowerMatch)); + || (secondaryPath != null && secondaryPath.toLowerCase().equals(lowerMatch)) + || (actionType != null && actionType.toLowerCase().equals(lowerMatch)) + || (additionalInfo != null && additionalInfo.toLowerCase().equals(lowerMatch)) + || (associatedRuleId != null && associatedRuleId.toLowerCase().equals(lowerMatch)) + || (reasonText != null && reasonText.toLowerCase().equals(lowerMatch)) + || (label != null && label.toLowerCase().equals(lowerMatch)) + || taintFlags.stream().anyMatch(flag -> flag.toLowerCase().equals(lowerMatch)) + || knowledge.values().stream().anyMatch(value -> value != null && value.toLowerCase().equals(lowerMatch)); } /** @@ -116,13 +141,356 @@ public boolean matchesPattern(Pattern pattern) { return false; } return (filePath != null && pattern.matcher(filePath).matches()) - || (secondaryPath != null && pattern.matcher(secondaryPath).matches()) - || (actionType != null && pattern.matcher(actionType).matches()) - || (additionalInfo != null && pattern.matcher(additionalInfo).matches()) - || (associatedRuleId != null && pattern.matcher(associatedRuleId).matches()) - || (reasonText != null && pattern.matcher(reasonText).matches()) - || (label != null && pattern.matcher(label).matches()) - || taintFlags.stream().anyMatch(flag -> pattern.matcher(flag).matches()) - || knowledge.values().stream().anyMatch(value -> value != null && pattern.matcher(value).matches()); + || (secondaryPath != null && pattern.matcher(secondaryPath).matches()) + || (actionType != null && pattern.matcher(actionType).matches()) + || (additionalInfo != null && pattern.matcher(additionalInfo).matches()) + || (associatedRuleId != null && pattern.matcher(associatedRuleId).matches()) + || (reasonText != null && pattern.matcher(reasonText).matches()) + || (label != null && pattern.matcher(label).matches()) + || taintFlags.stream().anyMatch(flag -> pattern.matcher(flag).matches()) + || knowledge.values().stream().anyMatch(value -> value != null && pattern.matcher(value).matches()); + } + + // ======================================== + // Constructors (Manual - Lombok @AllArgsConstructor removed for overloading) + // ======================================== + + /** + * Default constructor. + * Initializes collection fields to empty lists/maps. + */ + public Node() { + this.taintFlags = new ArrayList<>(); + this.knowledge = new HashMap<>(); + this.reasonTraceRefs = new ArrayList<>(); + this.reasonInlineTraces = new ArrayList<>(); + } + + /** + * Constructor for DOM parser - maintains backward compatibility. + * 22 parameters (original signature). + * + * Reason trace data is not needed as DOM parser uses rawNodePool (JAXB objects) + * to access Reason elements for building innerStackTrace. + * + * @param id Node ID + * @param filePath Source file path + * @param line Start line number + * @param lineEnd End line number + * @param colStart Start column + * @param colEnd End column + * @param contextId Context identifier + * @param snippet Code snippet + * @param actionType Action type + * @param additionalInfo Additional information + * @param associatedRuleId Associated rule ID + * @param taintFlags List of taint flags + * @param knowledge Knowledge map + * @param secondaryPath Secondary location path + * @param secondaryLine Secondary line number + * @param secondaryLineEnd Secondary end line + * @param secondaryColStart Secondary start column + * @param secondaryColEnd Secondary end column + * @param secondaryContextId Secondary context ID + * @param detailsOnly Details only flag + * @param label Node label + * @param reasonText Reason text summary + */ + public Node( + String id, String filePath, int line, int lineEnd, int colStart, int colEnd, + String contextId, String snippet, String actionType, String additionalInfo, + String associatedRuleId, List taintFlags, Map knowledge, + String secondaryPath, int secondaryLine, int secondaryLineEnd, + int secondaryColStart, int secondaryColEnd, String secondaryContextId, + boolean detailsOnly, String label, String reasonText + ) { + // Call extended constructor with empty lists for Reason trace data + this(id, filePath, line, lineEnd, colStart, colEnd, contextId, snippet, + actionType, additionalInfo, associatedRuleId, taintFlags, knowledge, + secondaryPath, secondaryLine, secondaryLineEnd, secondaryColStart, + secondaryColEnd, secondaryContextId, detailsOnly, label, reasonText, + new ArrayList<>(), new ArrayList<>()); + } + + /** + * Extended constructor for Streaming parser - includes Reason trace data. + * 24 parameters (includes reasonTraceRefs and reasonInlineTraces). + * + * Enables innerStackTrace building from parsed Reason elements. + * Streaming parser must store Reason trace data in Node fields since it doesn't + * have JAXB objects like DOM parser. + * + * @param id Node ID + * @param filePath Source file path + * @param line Start line number + * @param lineEnd End line number + * @param colStart Start column + * @param colEnd End column + * @param contextId Context identifier + * @param snippet Code snippet + * @param actionType Action type + * @param additionalInfo Additional information + * @param associatedRuleId Associated rule ID + * @param taintFlags List of taint flags + * @param knowledge Knowledge map + * @param secondaryPath Secondary location path + * @param secondaryLine Secondary line number + * @param secondaryLineEnd Secondary end line + * @param secondaryColStart Secondary start column + * @param secondaryColEnd Secondary end column + * @param secondaryContextId Secondary context ID + * @param detailsOnly Details only flag + * @param label Node label + * @param reasonText Reason text summary + * @param reasonTraceRefs List of TraceRef IDs from Reason element + * @param reasonInlineTraces List of inline Traces from Reason element + */ + public Node( + String id, String filePath, int line, int lineEnd, int colStart, int colEnd, + String contextId, String snippet, String actionType, String additionalInfo, + String associatedRuleId, List taintFlags, Map knowledge, + String secondaryPath, int secondaryLine, int secondaryLineEnd, + int secondaryColStart, int secondaryColEnd, String secondaryContextId, + boolean detailsOnly, String label, String reasonText, + List reasonTraceRefs, List reasonInlineTraces + ) { + this.id = id; + this.filePath = filePath; + this.line = line; + this.lineEnd = lineEnd; + this.colStart = colStart; + this.colEnd = colEnd; + this.contextId = contextId; + this.snippet = snippet; + this.actionType = actionType; + this.additionalInfo = additionalInfo; + this.associatedRuleId = associatedRuleId; + this.taintFlags = taintFlags != null ? taintFlags : new ArrayList<>(); + this.knowledge = knowledge != null ? knowledge : new HashMap<>(); + this.secondaryPath = secondaryPath; + this.secondaryLine = secondaryLine; + this.secondaryLineEnd = secondaryLineEnd; + this.secondaryColStart = secondaryColStart; + this.secondaryColEnd = secondaryColEnd; + this.secondaryContextId = secondaryContextId; + this.detailsOnly = detailsOnly; + this.label = label; + this.reasonText = reasonText; + this.reasonTraceRefs = reasonTraceRefs != null ? reasonTraceRefs : new ArrayList<>(); + this.reasonInlineTraces = reasonInlineTraces != null ? reasonInlineTraces : new ArrayList<>(); + } + + // ======================================== + // Inner Classes + // ======================================== + + @Getter + @Setter + public static class FPRInfo { + private String uuid; + private String buildId; + private String FPRName; + private String sourceBasePath; + private int numberOfFiles; + private int scanTime; + private FilterTemplate filterTemplate; + private FilterSet defaultEnabledFilterSet; + + public FPRInfo(Path extractedPath, Path FPRPath) { + FPRName = String.valueOf(FPRPath.getFileName()); + try { + // Use streaming parser for better memory efficiency + extractInfoFromAuditFvdlStreaming(extractedPath); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * Extract FPR metadata from audit.fvdl using DOM parsing (original method). + * Kept for backward compatibility. + * + * @param extractedPath Path to extracted FPR directory + * @throws Exception if parsing fails + */ + private void extractInfoFromAuditFvdl(Path extractedPath) throws Exception { + Path auditPath = extractedPath.resolve("audit.fvdl"); + + if (!Files.exists(auditPath)) { + throw new IllegalStateException("audit.fvdl not found in " + extractedPath); + } + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); + factory.setFeature("http://xml.org/sax/features/external-general-entities", false); + factory.setFeature("http://xml.org/sax/features/external-parameter-entities", false); + factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setFeature("http://xml.org/sax/features/validation", false); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document auditDoc = builder.parse(auditPath.toFile()); + + // Extract UUID + NodeList uuidNodes = auditDoc.getElementsByTagName("UUID"); + if (uuidNodes.getLength() > 0) { + this.uuid = uuidNodes.item(0).getTextContent(); + } + + // Extract Build information + NodeList buildNodes = auditDoc.getElementsByTagName("Build"); + if (buildNodes.getLength() > 0) { + Element buildElement = (Element) buildNodes.item(0); + this.buildId = getFirstElementContent(buildElement, "BuildID", ""); + this.sourceBasePath = getFirstElementContent(buildElement, "SourceBasePath", ""); + this.numberOfFiles = parseIntegerContent(getFirstElementContent(buildElement, "NumberFiles", "0")); + this.scanTime = parseIntegerContent(getFirstElementContent(buildElement, "ScanTime", "0")); + } + } + + /** + * Extract FPR metadata from audit.fvdl using streaming XML parsing (StAX). + * More memory-efficient than DOM parsing for large files. + * + * Extracts: + * - UUID + * - Build information (BuildID, SourceBasePath, NumberFiles, ScanTime) + * + * @param extractedPath Path to extracted FPR directory + * @throws Exception if parsing fails + */ + private void extractInfoFromAuditFvdlStreaming(Path extractedPath) throws Exception { + Path auditPath = extractedPath.resolve("audit.fvdl"); + + if (!Files.exists(auditPath)) { + throw new IllegalStateException("audit.fvdl not found in " + extractedPath); + } + + javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newInstance(); + // Security: Disable external entity processing + factory.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + factory.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, false); + + try (java.io.InputStream inputStream = Files.newInputStream(auditPath)) { + javax.xml.stream.XMLStreamReader reader = factory.createXMLStreamReader(inputStream); + + boolean inBuild = false; + String currentElement = null; + + while (reader.hasNext()) { + int event = reader.next(); + + if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("UUID".equals(localName)) { + // Extract UUID text content + this.uuid = readElementText(reader); + + } else if ("Build".equals(localName)) { + // Entering Build section + inBuild = true; + + } else if (inBuild) { + // Inside Build section, capture element name + currentElement = localName; + } + + } else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS && inBuild && currentElement != null) { + // Read text content for Build child elements + String text = reader.getText().trim(); + if (!text.isEmpty()) { + switch (currentElement) { + case "BuildID": + this.buildId = text; + break; + case "SourceBasePath": + this.sourceBasePath = text; + break; + case "NumberFiles": + this.numberOfFiles = parseIntegerContent(text); + break; + case "ScanTime": + this.scanTime = parseIntegerContent(text); + break; + } + } + + } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Build".equals(localName)) { + // Exiting Build section, we have all needed data + inBuild = false; + // Early exit: we've extracted all needed metadata + if (this.uuid != null) { + break; // Stop parsing, we have everything + } + + } else if (inBuild) { + // Clear current element when exiting child element + currentElement = null; + } + } + } + + reader.close(); + + } catch (javax.xml.stream.XMLStreamException e) { + throw new Exception("Failed to parse audit.fvdl using streaming parser", e); + } + } + + /** + * Helper method to read element text content using StAX reader. + * Advances reader to the text content and returns it. + * + * @param reader XMLStreamReader positioned at START_ELEMENT + * @return Text content of the element, or empty string if no text + * @throws javax.xml.stream.XMLStreamException if reading fails + */ + private String readElementText(javax.xml.stream.XMLStreamReader reader) throws javax.xml.stream.XMLStreamException { + StringBuilder text = new StringBuilder(); + while (reader.hasNext()) { + int event = reader.next(); + if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) { + text.append(reader.getText()); + } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) { + break; + } + } + return text.toString().trim(); + } + + private String getFirstElementContent(Element parent, String tagName, String defaultValue) { + NodeList nodes = parent.getElementsByTagName(tagName); + if (nodes != null && nodes.getLength() > 0 && nodes.item(0) != null) { + return nodes.item(0).getTextContent(); + } + return defaultValue; + } + + private int parseIntegerContent(String content) { + + if (StringUtil.isEmpty(content)) { + return 0; + } + + try { + return Integer.parseInt(content); + } catch (NumberFormatException e) { + System.err.println("Error parsing integer: " + content); + return 0; + } + } + + public Optional getDefaultEnabledFilterSet() { + if (filterTemplate == null || filterTemplate.getFilterSets() == null) { + return Optional.empty(); + } + + return filterTemplate.getFilterSets().stream() + .filter(FilterSet::isEnabled) + .findFirst(); + } } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedDescription.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedDescription.java new file mode 100644 index 00000000000..1af15e2bf57 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedDescription.java @@ -0,0 +1,63 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.model; + + +import lombok.Builder; +import lombok.Data; + +/** + * Streaming representation of a Description from FVDL. + * Replaces JAXB Description for streaming parsing. + * + * Structure matches JAXB Description class from Descriptions section: + * + * ... + * ... + * + */ +@Data +@Builder +public class StreamedDescription { + /** + * The classID attribute - unique identifier for this description. + * Used to map descriptions to vulnerabilities. + */ + private String classID; + + /** + * Short description text from element. + * Contains FVDL markup with Replace, Paragraph, IfDef, etc. tags. + */ + private String abstractText; + + /** + * Detailed explanation text from element. + * Contains FVDL markup with Replace, Paragraph, IfDef, etc. tags. + */ + private String explanation; + + /** + * Get abstract text (alias for compatibility with JAXB Description). + */ + public String getAbstract() { + return abstractText; + } + + /** + * Set abstract text (alias for compatibility with JAXB Description). + */ + public void setAbstract(String abstractText) { + this.abstractText = abstractText; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedTrace.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedTrace.java new file mode 100644 index 00000000000..b4258f58cae --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedTrace.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.model; + + + +import java.util.ArrayList; +import java.util.List; + +import lombok.Builder; +import lombok.Data; + +/** + * Streaming representation of a UnifiedTrace from FVDL. + * Replaces JAXB UnifiedTrace for streaming parsing. + * + * IMPORTANT: Entry now stores full Node objects to preserve inline nodes + * through Post-Processing (required for innerStackTrace building). + */ +@Data +@Builder +public class StreamedTrace { + private String id; + private Primary primary; + + /** + * Primary trace container with entries. + */ + @Data + @Builder + public static class Primary { + @Builder.Default + private List entries = new ArrayList<>(); + + /** + * Single trace entry - can contain inline Node or NodeRef. + * + * CHANGED: Now stores full Node object instead of just nodeId. + * This allows inline nodes (without IDs) to be preserved through Post-Processing. + * Previously, inline nodes were lost because NodePool lookup with null ID failed. + */ + @Data + @Builder + public static class Entry { + private String nodeId; // For backward compatibility and debugging (can be null for inline nodes) + private Node node; // CHANGED: Full Node object (from com.fortify.aviator.cli.fpr.models.Node) + private boolean isDefault; + + /** + * Check if this entry is a node reference (vs inline node). + */ + public boolean isNodeRef() { + return nodeId != null && !nodeId.isEmpty(); + } + + /** + * Check if this entry has a valid node object. + */ + public boolean hasNode() { + return node != null; + } + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedVulnerability.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedVulnerability.java new file mode 100644 index 00000000000..eb47b09bf02 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/StreamedVulnerability.java @@ -0,0 +1,137 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.model; + + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import lombok.Builder; +import lombok.Data; + +/** + * Lightweight vulnerability model for streaming parsing. + * Contains only essential fields to minimize memory footprint. + */ +@Data +@Builder +public class StreamedVulnerability { + + // Core identification + private String instanceId; + private String classId; + private String analyzerName; + + // Classification + private String kingdom; + private String type; + private String subType; + private String category; + + // Severity and risk + private Double defaultSeverity; + private Double instanceSeverity; + private Double confidence; + private Double impact; + private Double probability; + private String priority; + + // Location information + private String primaryFilePath; + private Integer primaryLine; + private Integer primaryLineEnd; + private Integer primaryColStart; + private Integer primaryColEnd; + + // Trace information (hierarchical structure matching FVDL schema) + // Each vulnerability can have multiple traces representing different data flow paths + @Builder.Default + private List traces = new ArrayList<>(); + + //private List files; + + // Metadata + @Builder.Default + private Map metadata = new HashMap<>(); + + // Description + private String shortDescription; + + // DAST-specific fields (if present) + private String requestMethod; + private String requestUrl; + private String attackPayload; + private String vulnerableParameter; + + // Auxiliary data and external entries (for DAST vulnerabilities) + @Builder.Default + private List> auxiliaryData = new ArrayList<>(); + + @Builder.Default + private List externalEntries = new ArrayList<>(); + + @Builder.Default + private Map externalIds = new HashMap<>(); + + // Replacement definitions from AnalysisInfo->Unified->ReplacementDefinitions + private ReplacementData replacementData; + + /** + * Represents an ExternalEntries Entry from FVDL. + * Simplified structure for streaming parsing. + */ + @Data + @Builder + public static class ExternalEntry { + private String url; + @Builder.Default + private List fields = new ArrayList<>(); + private String functionName; + private String functionNamespace; + private String locationPath; + private Integer locationLine; + private Integer locationColStart; + private Integer locationColEnd; + } + + /** + * Represents a Field within an ExternalEntry. + */ + @Data + @Builder + public static class EntryField { + private String name; + private String value; + private String type; + private String vulnTag; + } + + /** + * Represents a Trace in the FVDL structure. + * Matches the hierarchy: Vulnerability -> AnalysisInfo -> Unified -> Trace + * Each Trace contains a list of nodes representing a data flow path. + */ + @Data + @Builder + public static class Trace { + // Optional trace ID (used when trace is referenced from TracePool) + private String id; + + // List of nodes in this trace's data flow path + // Uses existing Node model from com.fortify.aviator.cli.fpr.models.Node + @Builder.Default + private List nodes = new ArrayList<>(); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java index 174f8a5fcca..17773d1de42 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/AuditProcessor.java @@ -57,6 +57,7 @@ import com.fortify.cli.aviator.config.TagMappingConfig; import com.fortify.cli.aviator.fpr.model.AuditIssue; import com.fortify.cli.aviator.fpr.model.FPRInfo; +import com.fortify.cli.aviator.fpr.utils.FileUtils; import com.fortify.cli.aviator.util.Constants; import com.fortify.cli.aviator.util.FprHandle; @@ -638,8 +639,7 @@ private String addCommentToIssueElement(Element issueElement, String commentText public File updateAndSaveAuditAndRemediationsXml(Map auditResponses, TagMappingConfig tagMappingConfig, - FPRInfo fprInfo, - FVDLProcessor fvdlProcessor) throws AviatorTechnicalException { + FPRInfo fprInfo) throws AviatorTechnicalException { // Step 1: Update the in-memory audit.xml document. This returns timestamps needed for remediations. Map remediationCommentTimestamps = updateAuditXml(auditResponses, tagMappingConfig); @@ -652,7 +652,7 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi // Step 3: Generate the in-memory remediations.xml document if needed. if (hasRemediations && !remediationCommentTimestamps.isEmpty()) { - this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo, fvdlProcessor); + this.remediationsDoc = generateRemediationsXml(auditResponses, remediationCommentTimestamps, fprInfo); } else { this.remediationsDoc = null; if (hasRemediations) { @@ -689,8 +689,7 @@ public File updateAndSaveAuditAndRemediationsXml(Map audi private Document generateRemediationsXml(Map auditResponses, Map remediationCommentTimestamps, - FPRInfo fprInfo, - FVDLProcessor fvdlProcessor) throws AviatorTechnicalException { + FPRInfo fprInfo) throws AviatorTechnicalException { try { DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); docFactory.setFeature(XMLConstants.FEATURE_SECURE_PROCESSING, true); @@ -753,7 +752,9 @@ private Document generateRemediationsXml(Map auditRespons filenameElement.setTextContent(filename); fileChangesElement.appendChild(filenameElement); - Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); + //Optional originalFileContentOptional = fvdlProcessor.getSourceFileContent(filename); + FileUtils fileUtils = new FileUtils(); + Optional originalFileContentOptional = fileUtils.getSourceFileContent(fprHandle, filename); if (originalFileContentOptional.isEmpty()) { logger.warn("WARN: Could not retrieve source code for file '{}'. Skipping remediation generation for this file for instanceId '{}'.", filename, instanceId); diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionParser.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionParser.java new file mode 100644 index 00000000000..50ff59c4db1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionParser.java @@ -0,0 +1,91 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static com.fortify.cli.aviator.fpr.processor.XmlParserUtils.readElementContentWithMarkup; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.model.*; + +/** + * Parses Description elements from FVDL XML. + * Handles Abstract and Explanation content with nested markup. + */ +public class DescriptionParser { + private static final Logger logger = LoggerFactory.getLogger(DescriptionParser.class); + + /** + * Parse a single Description element. + * + * @param reader XMLStreamReader positioned at Description start element + * @param classID The classID attribute value + * @return StreamedDescription object or null + */ + public StreamedDescription parseDescription(XMLStreamReader reader, String classID) throws XMLStreamException { + StreamedDescription.StreamedDescriptionBuilder builder = StreamedDescription.builder(); + builder.classID(classID); + + String abstractText = null; + String explanationText = null; + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Abstract".equals(localName)) { + abstractText = readElementContentWithMarkup(reader, "Abstract"); + continue; + + } else if ("Explanation".equals(localName)) { + explanationText = readElementContentWithMarkup(reader, "Explanation"); + continue; + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + } + } + + builder.abstractText(abstractText); + builder.explanation(explanationText); + + return builder.build(); + } + + /** + * Parse the entire Descriptions section. + */ + public void parseDescriptions(XMLStreamReader reader, java.util.Map descriptionCache) + throws XMLStreamException { + logger.debug("Streaming parsing of Descriptions"); + + String classID = reader.getAttributeValue(null, "classID"); + if (classID != null && !classID.isEmpty()) { + StreamedDescription description = parseDescription(reader, classID); + if (description != null) { + descriptionCache.put(classID, description); + } + } else { + logger.warn("Description missing classID, skipping"); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionProcessor.java index f1cde99c9e7..c516d129525 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/DescriptionProcessor.java @@ -23,7 +23,9 @@ import com.fortify.cli.aviator.fpr.Vulnerability; import com.fortify.cli.aviator.fpr.jaxb.Description; +import com.fortify.cli.aviator.fpr.model.FVDLMetadata; import com.fortify.cli.aviator.fpr.model.ReplacementData; +import com.fortify.cli.aviator.fpr.model.StreamedDescription; /** * Processor for FVDL Descriptions section. Caches descriptions by classID and processes @@ -77,6 +79,39 @@ public String[] processForVuln(Vulnerability vuln, String classId, ReplacementDa return new String[]{shortDesc, explanation}; } + /** + * Processes description for a Streaming vulnerability, applying replacements and conditionals. + * + * @param vuln Vulnerability object + * @param classId Class ID for description lookup + * @param replacementData Replacement data from AnalysisInfo + * @return Array of [shortDescription, explanation] + */ + + public String[] processForVuln(Vulnerability vuln, String classId, ReplacementData replacementData, FVDLMetadata streamingFvdlData) { + //Description desc = descriptionCache.get(classId); + StreamedDescription desc = streamingFvdlData.getDescriptionCache().get(classId); + if (desc == null) { + logger.debug("No description found for classID: {}", classId); + return new String[]{"", ""}; + } + + String abstractText = desc.getAbstract() != null ? desc.getAbstract() : ""; + String explanationText = desc.getExplanation() != null ? desc.getExplanation() : ""; + + + /*logger.info("For classId {} ", classId); + logger.info("Streaming abstractText {} ", abstractText); + logger.info("Streaming explanationText {} ", explanationText);*/ + + + // Use the new parser to process the text + String shortDesc = FvdlParser.parseAndRender(abstractText, vuln, replacementData); + String explanation = FvdlParser.parseAndRender(explanationText, vuln, replacementData); + + return new String[]{shortDesc, explanation}; + } + /** * A custom exception to signal that a replacement was required but not found. * This is used by ParagraphElement to fall back to AltParagraph. @@ -428,4 +463,4 @@ public static String parseAttribute(String attributeName, String content) { return content.substring(valueStart, valueEnd); } } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/IndexXMLProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/IndexXMLProcessor.java new file mode 100644 index 00000000000..53eddea1dc1 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/IndexXMLProcessor.java @@ -0,0 +1,94 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.util.Map; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; + +public class IndexXMLProcessor { + private static final Logger logger = LoggerFactory.getLogger(IndexXMLProcessor.class); + private final Map sourceFileMap; + private final Path extractedPath; + + + public IndexXMLProcessor(Path extractedPath, Map sourceFileMap) { + this.extractedPath = extractedPath; + this.sourceFileMap = sourceFileMap; + } + + /** + * Loads the source file map from FVDL. + */ + public void loadSourceFileMap() throws Exception { + Path srcArchiveDir = extractedPath.resolve("src-archive"); + Path indexPath = null; + + if (directoryContainsSourceFiles(srcArchiveDir)) { + indexPath = srcArchiveDir.resolve("index.xml"); + } + + if (indexPath == null) { + throw new NoSuchFileException("'src-archive' contained no source files under " + extractedPath); + } else if (!Files.exists(indexPath)) { + throw new NoSuchFileException("A source directory was found, but its 'index.xml' is missing at: " + indexPath); + } + + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false); + factory.setFeature("http://xml.org/sax/features/validation", false); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document indexDoc = builder.parse(indexPath.toFile()); + + NodeList entryNodes = indexDoc.getElementsByTagName("entry"); + for (int i = 0; i < entryNodes.getLength(); i++) { + Element entry = (Element) entryNodes.item(i); + String key = entry.getAttribute("key"); + String value = entry.getTextContent(); + sourceFileMap.put(key, value); + } + } + + private boolean directoryContainsSourceFiles(Path dirPath) throws IOException { + if (!Files.isDirectory(dirPath)) { + return false; + } + + try (DirectoryStream stream = Files.newDirectoryStream(dirPath)) { + for (Path path : stream) { + boolean isRegularFile = Files.isRegularFile(path); + boolean isNotIndexXml = !path.getFileName().toString().equals("index.xml"); + + if (isRegularFile && isNotIndexXml) { + return true; + } + } + } + + return false; + } + +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MemoryTracker.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MemoryTracker.java new file mode 100644 index 00000000000..a5ad58e2ce3 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MemoryTracker.java @@ -0,0 +1,269 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.Vulnerability; +import com.fortify.cli.aviator.fpr.model.*; + + +/** + * Tracks and logs memory consumption during FVDL parsing. + * Provides detailed metrics on JVM memory usage and data structure sizes. + */ +public class MemoryTracker { + private static final Logger logger = LoggerFactory.getLogger(MemoryTracker.class); + + private long peakMemoryBeforeParsing = 0; + private long peakMemoryPass1 = 0; + private long peakMemoryPass2 = 0; + private long peakMemoryPostProcessing = 0; + + /** + * Get current used memory without triggering GC. + */ + public long getCurrentUsedMemory() { + Runtime runtime = Runtime.getRuntime(); + return runtime.totalMemory() - runtime.freeMemory(); + } + + /** + * Initialize baseline memory before parsing. + */ + public void initializeBaseline() { + peakMemoryBeforeParsing = getCurrentUsedMemory(); + } + + /** + * Update pass 1 peak memory. + */ + public void updatePass1Peak() { + peakMemoryPass1 = Math.max(peakMemoryPass1, getCurrentUsedMemory()); + } + + /** + * Initialize pass 1 peak. + */ + public void initializePass1Peak() { + peakMemoryPass1 = getCurrentUsedMemory(); + } + + /** + * Update pass 2 peak memory. + */ + public void updatePass2Peak() { + peakMemoryPass2 = Math.max(peakMemoryPass2, getCurrentUsedMemory()); + } + + /** + * Initialize pass 2 peak. + */ + public void initializePass2Peak() { + peakMemoryPass2 = getCurrentUsedMemory(); + } + + /** + * Update post-processing peak memory. + */ + public void updatePostProcessingPeak() { + peakMemoryPostProcessing = Math.max(peakMemoryPostProcessing, getCurrentUsedMemory()); + } + + /** + * Initialize post-processing peak. + */ + public void initializePostProcessingPeak() { + peakMemoryPostProcessing = getCurrentUsedMemory(); + } + + public long getPeakMemoryPass1() { + return peakMemoryPass1; + } + + public long getPeakMemoryPass2() { + return peakMemoryPass2; + } + + public long getPeakMemoryPostProcessing() { + return peakMemoryPostProcessing; + } + + /** + * Log memory consumption with detailed metrics. + */ + public void logMemoryConsumption(String phase) { + logMemoryConsumptionWithPeak(phase, 0, null, null, null); + } + + /** + * Log memory consumption with peak tracking and data structure breakdown. + */ + public void logMemoryConsumptionWithPeak(String phase, long peakMemoryDuringPhase, + FVDLMetadata fvdlMetadata, + List rawVulnerabilities, + List vulnerabilities) { + Runtime runtime = Runtime.getRuntime(); + + // Force garbage collection for accurate readings + runtime.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + + long totalMemory = runtime.totalMemory(); + long freeMemory = runtime.freeMemory(); + long usedMemory = totalMemory - freeMemory; + long maxMemory = runtime.maxMemory(); + + double usedMemoryMB = usedMemory / (1024.0 * 1024.0); + double totalMemoryMB = totalMemory / (1024.0 * 1024.0); + double maxMemoryMB = maxMemory / (1024.0 * 1024.0); + double freeMemoryMB = freeMemory / (1024.0 * 1024.0); + double peakMemoryMB = peakMemoryDuringPhase / (1024.0 * 1024.0); + + double usedPercentage = (usedMemory * 100.0) / maxMemory; + double peakPercentage = (peakMemoryDuringPhase * 100.0) / maxMemory; + + logger.info("=== Memory Consumption {} ===", phase); + logger.info(" Used Memory: {} MB", String.format("%.2f", usedMemoryMB)); + logger.info(" Free Memory: {} MB", String.format("%.2f", freeMemoryMB)); + logger.info(" Total Memory: {} MB", String.format("%.2f", totalMemoryMB)); + logger.info(" Max Memory: {} MB", String.format("%.2f", maxMemoryMB)); + logger.info(" Usage: {}% of max memory", String.format("%.2f", usedPercentage)); + + if (peakMemoryDuringPhase > 0) { + logger.info(" Peak Memory: {} MB ({}% of max)", + String.format("%.2f", peakMemoryMB), + String.format("%.2f", peakPercentage)); + double peakDeltaMB = (peakMemoryDuringPhase - usedMemory) / (1024.0 * 1024.0); + logger.info(" Peak Delta: {} MB (peak was {} higher than current)", + String.format("%.2f", peakDeltaMB), + peakDeltaMB > 0 ? String.format("%.2f MB", peakDeltaMB) : "not higher"); + } + + if (fvdlMetadata != null) { + logDataStructureSizes(fvdlMetadata, rawVulnerabilities, vulnerabilities); + } + + logger.info("================================="); + } + + /** + * Log the sizes of key data structures. + */ + private void logDataStructureSizes(FVDLMetadata fvdlMetadata, + List rawVulnerabilities, + List vulnerabilities) { + logger.info(" --- Data Structure Sizes ---"); + + int nodePoolSize = fvdlMetadata.getNodePool().size(); + long estimatedNodePoolMemory = estimateNodePoolMemory(nodePoolSize); + logger.info(" NodePool: {} nodes (~{} MB)", + nodePoolSize, + String.format("%.2f", estimatedNodePoolMemory / (1024.0 * 1024.0))); + + int tracePoolSize = fvdlMetadata.getTracePool().size(); + long estimatedTracePoolMemory = estimateTracePoolMemory(tracePoolSize); + logger.info(" TracePool: {} traces (~{} MB)", + tracePoolSize, + String.format("%.2f", estimatedTracePoolMemory / (1024.0 * 1024.0))); + + int descriptionCacheSize = fvdlMetadata.getDescriptionCache().size(); + long estimatedDescriptionMemory = estimateDescriptionCacheMemory(descriptionCacheSize); + logger.info(" DescriptionCache: {} entries (~{} MB)", + descriptionCacheSize, + String.format("%.2f", estimatedDescriptionMemory / (1024.0 * 1024.0))); + + int ruleMetadataSize = fvdlMetadata.getRuleMetadata().size(); + long estimatedRuleMetadataMemory = estimateRuleMetadataMemory(ruleMetadataSize); + logger.info(" RuleMetadata: {} rules (~{} MB)", + ruleMetadataSize, + String.format("%.2f", estimatedRuleMetadataMemory / (1024.0 * 1024.0))); + + if (rawVulnerabilities != null && !rawVulnerabilities.isEmpty()) { + long estimatedVulnMemory = estimateVulnerabilitiesMemory(rawVulnerabilities.size()); + logger.info(" Raw Vulnerabilities: {} vulns (~{} MB)", + rawVulnerabilities.size(), + String.format("%.2f", estimatedVulnMemory / (1024.0 * 1024.0))); + } + + if (vulnerabilities != null && !vulnerabilities.isEmpty()) { + long estimatedEnrichedVulnMemory = estimateEnrichedVulnerabilitiesMemory(vulnerabilities.size()); + logger.info(" Enriched Vulnerabilities: {} vulns (~{} MB)", + vulnerabilities.size(), + String.format("%.2f", estimatedEnrichedVulnMemory / (1024.0 * 1024.0))); + } + + long totalEstimated = estimatedNodePoolMemory + estimatedTracePoolMemory + + estimatedDescriptionMemory + estimatedRuleMetadataMemory; + if (rawVulnerabilities != null && !rawVulnerabilities.isEmpty()) { + totalEstimated += estimateVulnerabilitiesMemory(rawVulnerabilities.size()); + } + if (vulnerabilities != null && !vulnerabilities.isEmpty()) { + totalEstimated += estimateEnrichedVulnerabilitiesMemory(vulnerabilities.size()); + } + + logger.info(" Total Estimated: ~{} MB", + String.format("%.2f", totalEstimated / (1024.0 * 1024.0))); + } + + private long estimateNodePoolMemory(int nodeCount) { + return (long) (nodeCount * 1.5 * 1024); + } + + private long estimateTracePoolMemory(int traceCount) { + return (long) (traceCount * 0.5 * 1024); + } + + private long estimateDescriptionCacheMemory(int descCount) { + return (long) (descCount * 3 * 1024); + } + + private long estimateRuleMetadataMemory(int ruleCount) { + return (long) (ruleCount * 0.5 * 1024); + } + + private long estimateVulnerabilitiesMemory(int vulnCount) { + return (long) (vulnCount * 3 * 1024); + } + + private long estimateEnrichedVulnerabilitiesMemory(int vulnCount) { + return (long) (vulnCount * 20 * 1024); + } + + public long estimateDescriptionCacheMemory(FVDLMetadata fvdlMetadata) { + return estimateDescriptionCacheMemory(fvdlMetadata.getDescriptionCache().size()); + } + + public long estimateRuleMetadataMemory(FVDLMetadata fvdlMetadata) { + return estimateRuleMetadataMemory(fvdlMetadata.getRuleMetadata().size()); + } + + public long estimateVulnerabilitiesMemory(List rawVulnerabilities) { + return estimateVulnerabilitiesMemory(rawVulnerabilities.size()); + } + + public long estimateNodePoolMemory(FVDLMetadata fvdlMetadata) { + return estimateNodePoolMemory(fvdlMetadata.getNodePool().size()); + } + + public long estimateTracePoolMemory(FVDLMetadata fvdlMetadata) { + return estimateTracePoolMemory(fvdlMetadata.getTracePool().size()); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MetadataParser.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MetadataParser.java new file mode 100644 index 00000000000..2ea579da8ff --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/MetadataParser.java @@ -0,0 +1,73 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static com.fortify.cli.aviator.fpr.processor.XmlParserUtils.readElementText; + +import java.util.HashMap; +import java.util.Map; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Parses metadata sections from FVDL XML. + * Handles EngineData with rule metadata. + */ +public class MetadataParser { + private static final Logger logger = LoggerFactory.getLogger(MetadataParser.class); + + /** + * Parse EngineData section for rule metadata. + * Collects Rule metadata with format: ruleId -> Map + */ + public void parseEngineData(XMLStreamReader reader, Map> ruleMetadata) + throws XMLStreamException { + logger.debug("Streaming EngineData parsing started"); + int depth = 1; + String currentRuleId = null; + Map currentMetadata = null; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Rule".equals(localName)) { + currentRuleId = reader.getAttributeValue(null, "id"); + currentMetadata = new HashMap<>(); + } else if ("Group".equals(localName) && currentMetadata != null) { + String groupName = reader.getAttributeValue(null, "name"); + String groupValue = readElementText(reader); + if (groupName != null && groupValue != null) { + currentMetadata.put(groupName, groupValue); + } + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("Rule".equals(reader.getLocalName()) && currentRuleId != null && currentMetadata != null) { + ruleMetadata.put(currentRuleId, currentMetadata); + currentRuleId = null; + currentMetadata = null; + } + } + } + logger.debug("Streaming Engine Data parsing completed"); + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/NodeParser.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/NodeParser.java new file mode 100644 index 00000000000..6281d17292f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/NodeParser.java @@ -0,0 +1,249 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static com.fortify.cli.aviator.fpr.processor.XmlParserUtils.*; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.model.*; + +/** + * Parses Node elements from FVDL XML. + * Handles both UnifiedNodePool nodes and inline nodes within traces. + */ +public class NodeParser { + private static final Logger logger = LoggerFactory.getLogger(NodeParser.class); + private final TraceParser traceParser; + + public NodeParser(TraceParser traceParser) { + this.traceParser = traceParser; + } + + /** + * Parse a Node element and create a Node object. + * + * @param reader XMLStreamReader positioned at Node start element + * @param passedNodeId Optional node ID (can be null, will read from attributes if needed) + * @return Parsed Node object, or null if parsing fails + */ + public Node parseNode(XMLStreamReader reader, String passedNodeId) throws XMLStreamException { + String nodeId = passedNodeId; + if (nodeId == null) { + nodeId = reader.getAttributeValue(null, "id"); + } + + if (nodeId == null) { + nodeId = "inline_" + System.nanoTime(); + logger.debug("Node has no ID attribute - generating temporary ID: {}", nodeId); + } + + logger.debug("Start parseNode for nodeId: {}", nodeId); + + String filePath = ""; + int line = 0; + int lineEnd = 0; + int colStart = 0; + int colEnd = 0; + String contextId = ""; + String snippet = ""; + String actionType = ""; + String additionalInfo = ""; + String ruleId = ""; + String label = reader.getAttributeValue(null, "label"); + if (label == null) label = ""; + + String detailsOnlyStr = reader.getAttributeValue(null, "detailsOnly"); + boolean detailsOnly = "true".equalsIgnoreCase(detailsOnlyStr); + + String secondaryPath = ""; + int secondaryLine = 0; + int secondaryLineEnd = 0; + int secondaryColStart = 0; + int secondaryColEnd = 0; + String secondaryContextId = ""; + + List taintFlags = new ArrayList<>(); + Map knowledge = new HashMap<>(); + StringBuilder reasonTextBuilder = new StringBuilder(); + + List reasonTraceRefs = new ArrayList<>(); + List reasonInlineTraces = new ArrayList<>(); + + logger.debug("Node {} - Initial attributes: label={}, detailsOnly={}", nodeId, label, detailsOnly); + + while (reader.hasNext()) { + int event = reader.next(); + + if((event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)) + logger.debug("[parseNode {}] Event: {}, LocalName: {}", nodeId, getEventTypeName(event), reader.getLocalName()); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("SourceLocation".equals(localName)) { + filePath = getAttributeOrDefault(reader, "path", ""); + snippet = ""; + line = parseIntOrDefault(reader, "line", 0); + lineEnd = parseIntOrDefault(reader, "lineEnd", line); + colStart = parseIntOrDefault(reader, "colStart", 0); + colEnd = parseIntOrDefault(reader, "colEnd", 0); + contextId = getAttributeOrDefault(reader, "contextId", ""); + logger.debug("Node {} - Parsed SourceLocation: path={}, line={}", nodeId, filePath, line); + + } else if ("SecondaryLocation".equals(localName)) { + secondaryPath = getAttributeOrDefault(reader, "path", ""); + secondaryLine = parseIntOrDefault(reader, "line", 0); + secondaryLineEnd = parseIntOrDefault(reader, "lineEnd", secondaryLine); + secondaryColStart = parseIntOrDefault(reader, "colStart", 0); + secondaryColEnd = parseIntOrDefault(reader, "colEnd", 0); + secondaryContextId = getAttributeOrDefault(reader, "contextId", ""); + logger.debug("Node {} - Parsed SecondaryLocation: path={}, line={}", nodeId, secondaryPath, secondaryLine); + + } else if ("Action".equals(localName)) { + actionType = getAttributeOrDefault(reader, "type", ""); + additionalInfo = readElementText(reader); + logger.debug("Node {} - Parsed Action: type={}", nodeId, actionType); + continue; + + } else if ("Fact".equals(localName)) { + String factType = reader.getAttributeValue(null, "type"); + String factValue = readElementText(reader); + if (factValue != null) { + if ("TaintFlags".equalsIgnoreCase(factType)) { + taintFlags.add(factValue); + } else { + knowledge.put(factType, factValue); + } + } + continue; + + } else if ("Rule".equals(localName)) { + String ruleID = reader.getAttributeValue(null, "ruleID"); + if (ruleID != null && ruleId.isEmpty()) { + ruleId = ruleID; + } + if (ruleID != null) { + reasonTextBuilder.append("Rule: ").append(ruleID).append("\n"); + } + + } else if ("Reason".equals(localName)) { + logger.debug("Node {} - Parsing Reason element", nodeId); + ruleId = parseReasonElement(reader, reasonTraceRefs, reasonInlineTraces, + reasonTextBuilder, ruleId, nodeId); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if ("Node".equals(reader.getLocalName())) { + logger.debug("Node {} - Parsing complete", nodeId); + break; + } + } + } + + return new Node( + nodeId, filePath, line, lineEnd, colStart, colEnd, contextId, snippet, + actionType, additionalInfo, ruleId, taintFlags, knowledge, + secondaryPath, secondaryLine, secondaryLineEnd, secondaryColStart, + secondaryColEnd, secondaryContextId, detailsOnly, label, + reasonTextBuilder.toString().trim(), reasonTraceRefs, reasonInlineTraces + ); + } + + /** + * Parse Reason element for innerStackTrace support. + */ + private String parseReasonElement( + XMLStreamReader reader, + List reasonTraceRefs, + List reasonInlineTraces, + StringBuilder reasonTextBuilder, + String currentRuleId, + String parentNodeId) throws XMLStreamException { + + logger.debug("[parseReasonElement] Start parsing Reason for node: {}", parentNodeId); + String ruleId = currentRuleId; + int depth = 1; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Trace".equals(localName)) { + String traceId = reader.getAttributeValue(null, "id"); + StreamedTrace inlineTrace = traceParser.parseStreamedTrace(reader, traceId); + if (inlineTrace != null) { + reasonInlineTraces.add(inlineTrace); + } + continue; + + } else if ("TraceRef".equals(localName)) { + String refId = reader.getAttributeValue(null, "id"); + if (refId != null && !refId.isEmpty()) { + reasonTraceRefs.add(refId); + } + + } else if ("Rule".equals(localName)) { + String ruleID = reader.getAttributeValue(null, "ruleID"); + if (ruleID != null) { + if (ruleId == null || ruleId.isEmpty()) { + ruleId = ruleID; + } + reasonTextBuilder.append("Rule: ").append(ruleID).append("\n"); + } + } + depth++; + + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("Reason".equals(reader.getLocalName()) && depth == 0) { + return ruleId; + } + } + } + return ruleId; + } + + /** + * Create a placeholder Node for forward references. + */ + public Node createPlaceholderNode(String nodeId) { + return new Node( + nodeId, "", 0, 0, 0, 0, "", "", "", "", "", + new ArrayList<>(), new HashMap<>(), "", 0, 0, 0, 0, "", + false, "", "", new ArrayList<>(), new ArrayList<>() + ); + } + + private String getAttributeOrDefault(XMLStreamReader reader, String attrName, String defaultValue) { + String value = reader.getAttributeValue(null, attrName); + return value != null ? value : defaultValue; + } + + private int parseIntOrDefault(XMLStreamReader reader, String attrName, int defaultValue) { + String value = reader.getAttributeValue(null, attrName); + Integer parsed = parseIntSafe(value); + return parsed != null ? parsed : defaultValue; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java new file mode 100644 index 00000000000..bb8f8cdda53 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/StreamingFVDLProcessor.java @@ -0,0 +1,1660 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static com.fortify.cli.aviator.fpr.processor.XmlParserUtils.*; + +import java.io.IOException; +import java.io.InputStream; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.Vulnerability; +import com.fortify.cli.aviator.fpr.filter.AnalyzerType; +import com.fortify.cli.aviator.fpr.model.*; +import com.fortify.cli.aviator.fpr.utils.FileUtils; +import com.fortify.cli.aviator.fpr.utils.XmlUtils; +import com.fortify.cli.aviator.util.FprHandle; +import com.fortify.cli.aviator.util.StringUtil; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@AllArgsConstructor +public class StreamingFVDLProcessor { + private static final Logger logger = LoggerFactory.getLogger(StreamingFVDLProcessor.class); + private final XMLInputFactory xmlInputFactory; + @Getter + private final FVDLMetadata fvdlMetadata; + private final VulnFinalizer vulnFinalizer; + private final FileUtils fileUtils; + private final DescriptionProcessor descriptionProcessor; + @Getter + private final Map sourceFileMap; + private final List rawVulnerabilities; + @Getter + private final List vulnerabilities; + private final FprHandle fprHandle; + /*private final Path extractedPath; + private final IndexXMLProcessor indexXMLProcessor;*/ + + // Specialized parsers + private final MemoryTracker memoryTracker; + private final NodeParser nodeParser; + private final TraceParser traceParser; + private final DescriptionParser descriptionParser; + private final MetadataParser metadataParser; + + // Peak memory tracking + private long peakMemoryBeforeParsing = 0; + private long peakMemoryPass1 = 0; + private long peakMemoryPass2 = 0; + private long peakMemoryPostProcessing = 0; + + public StreamingFVDLProcessor(FprHandle fprHandle){ + this.vulnFinalizer = new VulnFinalizer(); + this.fileUtils = new FileUtils(); + this.fprHandle = fprHandle; + this.sourceFileMap = fprHandle.getSourceFileMap(); + this.xmlInputFactory = XMLInputFactory.newInstance(); + // Security: Disable external entity processing + xmlInputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false); + xmlInputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false); + //this.parsingMetadata = new ParsingMetadata(); + this.fvdlMetadata = new FVDLMetadata(); + this.rawVulnerabilities = new ArrayList<>(); + this.vulnerabilities = new ArrayList<>(); + this.descriptionProcessor = new DescriptionProcessor(); + + // Initialize specialized parsers + this.memoryTracker = new MemoryTracker(); + this.traceParser = new TraceParser(fvdlMetadata.getNodePool()); + this.nodeParser = new NodeParser(traceParser); + this.traceParser.setNodeParser(nodeParser); // Circular dependency for Reason parsing + this.descriptionParser = new DescriptionParser(); + this.metadataParser = new MetadataParser(); + /*this.extractedPath = extractedPath; + this.indexXMLProcessor = new IndexXMLProcessor(extractedPath, sourceFileMap);*/ + } + + + /** + * Parse an FPR file using two-pass parsing strategy. + * + * TWO-PASS PARSING STRATEGY: + * Pass 1: Parse metadata and pools (NodePool, TracePool, Descriptions, EngineData) + * This populates the reference pools needed for vulnerability parsing. + * Pass 2: Parse Vulnerabilities with fully populated pools + * NodeRef lookups now succeed because NodePool is populated. + * + * @param zipFile zipFile for FPR file + * @param entryName entry name for FVDL file + * @throws Exception If file access or parsing fails + */ + public void parse(ZipFile zipFile, String entryName) throws Exception { + logger.info("=== Two-Pass Streaming Parsing Started for FVDL file ==="); + + // Initialize memory tracking + memoryTracker.initializeBaseline(); + memoryTracker.logMemoryConsumption("Before Parsing (Baseline)"); + + /*logger.info("Loading source file map"); + //indexXMLProcessor.loadSourceFileMap();*/ + + // Update peak after loading source file map + memoryTracker.updatePass1Peak(); + + + ZipEntry fvdlEntry = zipFile.getEntry(entryName); + if (fvdlEntry == null) { + throw new IOException("audit.fvdl not found in FPR file"); + } + + try { + // ======================================== + // PASS 1: Parse Metadata and Pools + // ======================================== + logger.info(">>> PASS 1: Parsing metadata and pools (NodePool, TracePool, Descriptions, EngineData)"); + long pass1Start = System.currentTimeMillis(); + memoryTracker.initializePass1Peak(); + + try (InputStream is1 = zipFile.getInputStream(fvdlEntry)) { + parseMetadataAndPools(is1); + memoryTracker.updatePass1Peak(); + } + + long pass1Time = System.currentTimeMillis() - pass1Start; + logger.info("<<< PASS 1 Complete - Time: {}ms", pass1Time); + + memoryTracker.logMemoryConsumptionWithPeak("After PASS 1", memoryTracker.getPeakMemoryPass1(), + fvdlMetadata, rawVulnerabilities, vulnerabilities); + + // ======================================== + // PASS 2: Parse Vulnerabilities + // ======================================== + logger.info(">>> PASS 2: Parsing vulnerabilities (with populated pools)"); + long pass2Start = System.currentTimeMillis(); + memoryTracker.initializePass2Peak(); + + try (InputStream is2 = zipFile.getInputStream(fvdlEntry)) { + parseVulnerabilitiesOnly(is2); + memoryTracker.updatePass2Peak(); + } + + long pass2Time = System.currentTimeMillis() - pass2Start; + logger.info("<<< PASS 2 Complete - Time: {}ms", pass2Time); + logger.info(" - Vulnerabilities parsed: {}", rawVulnerabilities.size()); + memoryTracker.logMemoryConsumptionWithPeak("After PASS 2", memoryTracker.getPeakMemoryPass2(), + fvdlMetadata, rawVulnerabilities, vulnerabilities); + + // ======================================== + // Post-Processing: Enrich Vulnerabilities (Batch Processing) + // ======================================== + logger.info(">>> Post-Processing: Enriching vulnerabilities in batches"); + long processStart = System.currentTimeMillis(); + memoryTracker.initializePostProcessingPeak(); + + int batchSize = 1000; + int totalVulns = rawVulnerabilities.size(); + int totalBatches = (int) Math.ceil((double) totalVulns / batchSize); + + for (int batchNum = 0; batchNum < totalBatches; batchNum++) { + int start = batchNum * batchSize; + int end = Math.min(start + batchSize, totalVulns); + // Process current batch + for (int i = start; i < end; i++) { + StreamedVulnerability rawVuln = rawVulnerabilities.get(i); + Vulnerability enrichedVuln = processVulnerability(rawVuln, vulnFinalizer); + vulnerabilities.add(enrichedVuln); + + // Null out processed raw vulnerability to enable GC + rawVulnerabilities.set(i, null); + } + + // Track peak memory every 10 batches + if (batchNum % 10 == 0) { + memoryTracker.updatePostProcessingPeak(); + } + } + + // Final peak check + memoryTracker.updatePostProcessingPeak(); + + long processTime = System.currentTimeMillis() - processStart; + logger.info("<<< Post-Processing Complete - Time: {}ms", processTime); + logger.info(" - Processed {} vulnerabilities in {} batches", totalVulns, totalBatches); + memoryTracker.logMemoryConsumptionWithPeak("After Post-Processing", + memoryTracker.getPeakMemoryPostProcessing(), fvdlMetadata, rawVulnerabilities, vulnerabilities); + + // ======================================== + // Memory Optimization: Clear DescriptionCache and RuleMetadata + // ======================================== + logger.info(">>> Memory Optimization: Clearing DescriptionCache and RuleMetadata"); + + long descCount = fvdlMetadata.getDescriptionCache().size(); + long descMemoryEstimate = memoryTracker.estimateDescriptionCacheMemory(fvdlMetadata); + fvdlMetadata.getDescriptionCache().clear(); + logger.info(" - Cleared {} descriptions (~{} MB)", + descCount, String.format("%.2f", descMemoryEstimate / (1024.0 * 1024.0))); + + long ruleCount = fvdlMetadata.getRuleMetadata().size(); + long ruleMemoryEstimate = memoryTracker.estimateRuleMetadataMemory(fvdlMetadata); + fvdlMetadata.getRuleMetadata().clear(); + logger.info(" - Cleared {} rule metadata entries (~{} KB)", + ruleCount, String.format("%.2f", ruleMemoryEstimate / 1024.0)); + + memoryTracker.logMemoryConsumption("After Clearing Metadata Caches"); + + // ======================================== + // Memory Optimization: Clear Raw Vulnerabilities + // ======================================== + logger.info(">>> Memory Optimization: Clearing raw vulnerabilities list"); + long rawVulnCount = rawVulnerabilities.size(); + long rawVulnMemoryEstimate = memoryTracker.estimateVulnerabilitiesMemory(rawVulnerabilities); + rawVulnerabilities.clear(); + logger.info(" - Cleared {} raw vulnerabilities (~{} MB)", + rawVulnCount, String.format("%.2f", rawVulnMemoryEstimate / (1024.0 * 1024.0))); + memoryTracker.logMemoryConsumption("After Clearing Raw Vulnerabilities"); + + // ======================================== + // OPTIONAL: Clear NodePool and TracePool + // ======================================== + logger.info(">>> Memory Optimization: Clearing NodePool and TracePool"); + long nodePoolSize = fvdlMetadata.getNodePool().size(); + long tracePoolSize = fvdlMetadata.getTracePool().size(); + long nodePoolMemoryEstimate = memoryTracker.estimateNodePoolMemory(fvdlMetadata); + long tracePoolMemoryEstimate = memoryTracker.estimateTracePoolMemory(fvdlMetadata); + fvdlMetadata.getNodePool().clear(); + fvdlMetadata.getTracePool().clear(); + logger.info(" - Cleared NodePool ({} nodes, ~{} MB) and TracePool ({} traces, ~{} MB)", + nodePoolSize, String.format("%.2f", nodePoolMemoryEstimate / (1024.0 * 1024.0)), + tracePoolSize, String.format("%.2f", tracePoolMemoryEstimate / (1024.0 * 1024.0))); + memoryTracker.logMemoryConsumption("After Clearing Pools"); + + // ======================================== + // Summary + // ======================================== + long totalTime = pass1Time + pass2Time + processTime; + logger.info("=== Two-Pass Parsing Complete ==="); + logger.info(" Total Time: {}ms (Pass1: {}ms, Pass2: {}ms, Process: {}ms)", + totalTime, pass1Time, pass2Time, processTime); + logger.info(" Overhead: ~{:.1f}% compared to single-pass estimate", + ((pass1Time + pass2Time) / (double)(pass1Time + pass2Time) * 100) - 100); + + } catch (XMLStreamException e) { + throw new IOException("Failed to parse FVDL: " + e.getMessage(), e); + } + + } + + + /** + * PASS 1: Parse metadata and pools only. + * Skips Vulnerabilities section for later processing in Pass 2. + * + * Parsed sections: + * - EngineData (rule metadata) + * - UnifiedNodePool (node definitions) + * - UnifiedTracePool (trace definitions) + * - Description (vulnerability descriptions) + * - Build (skipped) + * + * Skipped sections: + * - Vulnerabilities (will be parsed in Pass 2) + */ + private void parseMetadataAndPools(InputStream is) throws XMLStreamException { + logger.debug("Pass 1: Starting metadata and pools parsing"); + XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(is); + + try { + while (reader.hasNext()) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + switch (localName) { + case "EngineData": + logger.debug("Pass 1: Parsing EngineData"); + parseEngineData(reader); + break; + case "Build": + logger.debug("Pass 1: Skipping Build"); + // Build is already skipped in parseEngineData + break; + case "UnifiedNodePool": + logger.debug("Pass 1: Parsing UnifiedNodePool"); + parseNodePool(reader); + break; + case "UnifiedTracePool": + logger.debug("Pass 1: Parsing UnifiedTracePool"); + parseTracePool(reader); + break; + case "Description": + logger.debug("Pass 1: Parsing Description"); + parseDescriptions(reader); + break; + case "Vulnerabilities": + logger.debug("Pass 1: Skipping Vulnerabilities (will parse in Pass 2)"); + skipSection(reader, "Vulnerabilities"); + break; + } + } + } + } finally { + reader.close(); + } + logger.debug("Pass 1: Metadata and pools parsing complete"); + } + + /** + * PASS 2: Parse vulnerabilities only. + * Skips all other sections that were already parsed in Pass 1. + * + * At this point, NodePool and TracePool are fully populated, + * so all NodeRef lookups will succeed. + * + * Parsed sections: + * - Vulnerabilities (with NodeRef resolution) + */ + private void parseVulnerabilitiesOnly(InputStream is) throws XMLStreamException { + logger.debug("Pass 2: Starting vulnerabilities parsing"); + XMLStreamReader reader = xmlInputFactory.createXMLStreamReader(is); + + try { + while (reader.hasNext()) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + switch (localName) { + case "Vulnerabilities": + logger.debug("Pass 2: Parsing Vulnerabilities"); + parseVulnerabilities(reader); + break; + case "EngineData": + case "Build": + case "UnifiedNodePool": + case "UnifiedTracePool": + case "Description": + logger.debug("Pass 2: Skipping {} (already parsed in Pass 1)", localName); + skipSection(reader, localName); + break; + } + } + } + } finally { + reader.close(); + } + logger.debug("Pass 2: Vulnerabilities parsing complete"); + } + + /** + * Parse EngineData section for rule metadata. + * Delegates to MetadataParser. + */ + private void parseEngineData(XMLStreamReader reader) throws XMLStreamException { + metadataParser.parseEngineData(reader, fvdlMetadata.getRuleMetadata()); + } + + /** + * Parse UnifiedNodePool section. + * Delegates node parsing to NodeParser. + */ + private void parseNodePool(XMLStreamReader reader) throws XMLStreamException { + logger.debug("Start parse UnifiedNodePool"); + + while (reader.hasNext()) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + if ("Node".equals(reader.getLocalName())) { + String nodeId = reader.getAttributeValue(null, "id"); + logger.debug("Found Node with ID: {}", nodeId); + + // Delegate to NodeParser + Node node = nodeParser.parseNode(reader, null); + + if (node != null) { + fvdlMetadata.getNodePool().put(node.getId(), node); + fvdlMetadata.setTotalNodes(fvdlMetadata.getTotalNodes() + 1); + logger.debug("Successfully added node {} to pool. Total nodes: {}", node.getId(), fvdlMetadata.getNodePool().size()); + } + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if ("UnifiedNodePool".equals(reader.getLocalName())) { + break; + } + } + } + logger.info("Nodes processed: {} ", fvdlMetadata.getNodePool().size()); + } + + /** + * Parse UnifiedTracePool section. + * Delegates to TraceParser. + */ + private void parseTracePool(XMLStreamReader reader) throws XMLStreamException { + logger.debug("start Unified Trace pool"); + + while (reader.hasNext()) { + int event = reader.next(); + + if((event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)) + logger.debug("[parseTracePool] Event: {}, LocalName: {}", getEventTypeName(event), reader.getLocalName()); + + if (event == XMLStreamConstants.START_ELEMENT) { + if ("Trace".equals(reader.getLocalName())) { + String traceId = reader.getAttributeValue(null, "id"); + logger.debug("Trace Id {} ", traceId); + if (traceId != null && !traceId.isEmpty()) { + logger.debug("TraceId not empty and not null {} ", traceId ); + // Delegate to TraceParser + StreamedTrace trace = + traceParser.parseStreamedTrace(reader, traceId); + if (trace != null) { + fvdlMetadata.getTracePool().put(traceId, trace); + fvdlMetadata.setTotalTraces(fvdlMetadata.getTotalTraces() + 1); + logger.debug("Successfully added trace {} to pool. Total traces: {}", traceId, fvdlMetadata.getTracePool().size()); + } + } else { + logger.warn("Trace missing or invalid ID, skipping"); + } + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if ("UnifiedTracePool".equals(reader.getLocalName())) { + logger.debug("Reached end of UnifiedTracePool, exiting loop"); + break; + } + } + } + logger.debug("Trace processed are {} ", fvdlMetadata.getTracePool().size()); + } + + /** + * Parse Descriptions section for vulnerability descriptions. + * Delegates to DescriptionParser. + */ + private void parseDescriptions(XMLStreamReader reader) throws XMLStreamException { + descriptionParser.parseDescriptions(reader, fvdlMetadata.getDescriptionCache()); + } + + private void parseVulnerabilities(XMLStreamReader reader) throws XMLStreamException { + logger.info("Start parse Vulnerabilities"); + + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + if ("Vulnerability".equals(reader.getLocalName())) { + StreamedVulnerability vuln = parseVulnerability(reader); + rawVulnerabilities.add(vuln); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if ("Vulnerabilities".equals(reader.getLocalName())) { + return; + } + } + } + } + + private StreamedVulnerability parseVulnerability(XMLStreamReader reader) throws XMLStreamException { + logger.debug("=== START parseVulnerability ==="); + StreamedVulnerability.StreamedVulnerabilityBuilder builder = StreamedVulnerability.builder(); + List traces = new ArrayList<>(); + List> auxiliaryDataList = new ArrayList<>(); + List externalEntriesList = new ArrayList<>(); + Map externalIdsMap = new HashMap<>(); + + String currentSection = null; + logger.debug("[parseVulnerability] Initialized traces list: {}", traces.size()); + + // Process until we hit the CLOSING tag + while (reader.hasNext()) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + logger.debug("[parseVulnerability] START_ELEMENT: {}", localName); + + switch (localName) { + // ...existing code... + case "ClassInfo": + currentSection = "ClassInfo"; + break; + case "InstanceInfo": + currentSection = "InstanceInfo"; + break; + case "AnalysisInfo": + currentSection = "AnalysisInfo"; + break; + case "ClassID": + if ("ClassInfo".equals(currentSection)) { + builder.classId(readElementText(reader)); + } + break; + case "Kingdom": + builder.kingdom(readElementText(reader)); + break; + case "Type": + builder.type(readElementText(reader)); + break; + case "Subtype": + builder.subType(readElementText(reader)); + break; + case "AnalyzerName": + builder.analyzerName(readElementText(reader)); + break; + case "DefaultSeverity": + builder.defaultSeverity(parseDoubleSafe(readElementText(reader))); + break; + case "InstanceID": + builder.instanceId(readElementText(reader)); + break; + case "InstanceSeverity": + builder.instanceSeverity(parseDoubleSafe(readElementText(reader))); + break; + case "MetaInfo": + // Parse MetaInfo element - contains instance-specific metadata overrides + parseMetaInfo(reader, builder); + break; + case "Confidence": + builder.confidence(parseDoubleSafe(readElementText(reader))); + break; + case "Trace": + logger.debug("[parseVulnerability] Found Trace element. Current traces count: {}", traces.size()); + StreamedVulnerability.Trace trace = parseTrace(reader); + if (trace != null) { + traces.add(trace); + logger.debug("[parseVulnerability] Added trace with {} nodes. Total traces: {}", + trace.getNodes().size(), traces.size()); + } + break; + case "ReplacementDefinitions": + // Parse ReplacementDefinitions from AnalysisInfo->Unified->ReplacementDefinitions + builder.replacementData(parseReplacementDefinitions(reader)); + break; + case "AuxiliaryData": + parseAuxiliaryData(reader, auxiliaryDataList); + break; + case "ExternalEntries": + parseExternalEntries(reader, externalEntriesList); + break; + case "ExternalID": + parseExternalID(reader, externalIdsMap); + break; + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + String localName = reader.getLocalName(); + + // Exit when we close the Vulnerability element + if ("Vulnerability".equals(localName)) { + break; + } + + // Reset section tracking + if ("ClassInfo".equals(localName) || + "InstanceInfo".equals(localName) || + "AnalysisInfo".equals(localName)) { + currentSection = null; + } + } + } + + int totalNodes = traces.stream().mapToInt(t -> t.getNodes() != null ? t.getNodes().size() : 0).sum(); + logger.debug("[parseVulnerability] Building vulnerability - traces: {}, total nodes: {}", traces.size(), totalNodes); + builder.traces(traces); + builder.auxiliaryData(auxiliaryDataList); + builder.externalEntries(externalEntriesList); + builder.externalIds(externalIdsMap); + + StreamedVulnerability vuln = builder.build(); + int vulnTotalNodes = vuln.getTraces() != null ? + vuln.getTraces().stream().mapToInt(t -> t.getNodes() != null ? t.getNodes().size() : 0).sum() : 0; + logger.debug("[parseVulnerability] Built vulnerability - traces: {}, total nodes: {}", + vuln.getTraces() != null ? vuln.getTraces().size() : "NULL", vulnTotalNodes); + + // Check which fields are populated + // VulnerabilityLoggingUtils.logPopulatedFields(vuln); + + //System.out.println("Traces count: " + vuln.getTraces().size()); + //populateFilesForVulnerability(vuln, fprPath, sourceFileMap, fileUtils); + //return builder.build(); + return vuln; + } + + + /** + * Parse MetaInfo element and extract metadata groups. + * MetaInfo structure: value... + * This is used for instance-specific metadata overrides in InstanceInfo section. + * + * @param reader XMLStreamReader positioned at MetaInfo start element + * @param builder StreamedVulnerability builder to populate metadata + * @throws XMLStreamException If XML parsing fails + */ + private void parseMetaInfo(XMLStreamReader reader, StreamedVulnerability.StreamedVulnerabilityBuilder builder) + throws XMLStreamException { + + Map instanceMetadata = new HashMap<>(); + int depth = 1; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Group".equals(localName)) { + // Get the "name" attribute + String groupName = reader.getAttributeValue(null, "name"); + + // Get the text content (value) + String groupValue = readElementText(reader); + + if (groupName != null && groupValue != null) { + instanceMetadata.put(groupName, groupValue.trim()); + logger.trace("Parsed MetaInfo Group: '{}' = '{}'", groupName, groupValue); + } + continue; // Skip depth++ since we consumed the element text + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("MetaInfo".equals(reader.getLocalName()) && depth == 0) { + break; // Exit when we close the MetaInfo element + } + } + } + + // Store the parsed metadata in the builder + if (!instanceMetadata.isEmpty()) { + builder.metadata(instanceMetadata); + logger.debug("Parsed {} MetaInfo groups", instanceMetadata.size()); + } + } + + /** + * Parse inline Trace from Vulnerability. + * Delegates to TraceParser. + */ + private StreamedVulnerability.Trace parseTrace(XMLStreamReader reader) throws XMLStreamException { + return traceParser.parseTrace(reader); + } + + /** + * Parse AuxiliaryData element. + * Structure: + * Replicates AuxiliaryProcessor logic for streaming parsing. + * + * @param reader XMLStreamReader positioned at AuxiliaryData start element + * @param auxiliaryDataList List to add parsed auxiliary data map to + */ + private void parseAuxiliaryData(XMLStreamReader reader, List> auxiliaryDataList) throws XMLStreamException { + Map auxMap = new HashMap<>(); + + // Get contentType attribute + String contentType = reader.getAttributeValue(null, "contentType"); + auxMap.put("contentType", contentType != null ? contentType : ""); + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("AuxField".equals(localName)) { + // Get AuxField attributes + String fieldName = reader.getAttributeValue(null, "name"); + String fieldValue = reader.getAttributeValue(null, "value"); + + if (fieldName != null) { + auxMap.put(fieldName, fieldValue != null ? fieldValue : ""); + } + + // Check for nested SourceLocation + parseNestedSourceLocation(reader, auxMap); + continue; // Skip depth++ since we handled the element + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + } + } + + auxiliaryDataList.add(auxMap); + } + + /** + * Parse nested SourceLocation within AuxField. + * Adds location attributes to the map with "loc" prefix. + * + * @param reader XMLStreamReader positioned after AuxField start + * @param auxMap Map to add location data to + */ + private void parseNestedSourceLocation(XMLStreamReader reader, Map auxMap) throws XMLStreamException { + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + if ("SourceLocation".equals(reader.getLocalName())) { + String path = reader.getAttributeValue(null, "path"); + String line = reader.getAttributeValue(null, "line"); + String colStart = reader.getAttributeValue(null, "colStart"); + String colEnd = reader.getAttributeValue(null, "colEnd"); + + auxMap.put("locPath", path != null ? path : ""); + auxMap.put("locLine", line != null ? line : "0"); + auxMap.put("locColStart", colStart != null ? colStart : "0"); + auxMap.put("locColEnd", colEnd != null ? colEnd : "0"); + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("AuxField".equals(reader.getLocalName())) { + return; // Exit when we close AuxField + } + } + } + } + + /** + * Parse ExternalEntries element. + * Structure: ... + * Replicates AuxiliaryProcessor ExternalEntries logic. + * + * @param reader XMLStreamReader positioned at ExternalEntries start element + * @param externalEntriesList List to add parsed entries to + */ + private void parseExternalEntries(XMLStreamReader reader, List externalEntriesList) + throws XMLStreamException { + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + if ("Entry".equals(reader.getLocalName())) { + StreamedVulnerability.ExternalEntry entry = parseExternalEntry(reader); + if (entry != null) { + externalEntriesList.add(entry); + } + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + } + } + } + + /** + * Parse a single Entry element within ExternalEntries. + * + * @param reader XMLStreamReader positioned at Entry start element + * @return Parsed ExternalEntry or null + */ + private StreamedVulnerability.ExternalEntry parseExternalEntry(XMLStreamReader reader) throws XMLStreamException { + StreamedVulnerability.ExternalEntry.ExternalEntryBuilder builder = + StreamedVulnerability.ExternalEntry.builder(); + + List fields = new ArrayList<>(); + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + switch (localName) { + case "URL": + builder.url(readElementText(reader)); + continue; + case "Function": + // Parse Function element attributes + String funcName = reader.getAttributeValue(null, "name"); + String funcNamespace = reader.getAttributeValue(null, "namespace"); + builder.functionName(funcName); + builder.functionNamespace(funcNamespace); + break; + case "SourceLocation": + // Parse SourceLocation attributes + String path = reader.getAttributeValue(null, "path"); + String line = reader.getAttributeValue(null, "line"); + String colStart = reader.getAttributeValue(null, "colStart"); + String colEnd = reader.getAttributeValue(null, "colEnd"); + builder.locationPath(path); + builder.locationLine(parseIntSafe(line)); + builder.locationColStart(parseIntSafe(colStart)); + builder.locationColEnd(parseIntSafe(colEnd)); + break; + case "Fields": + parseEntryFields(reader, fields); + continue; + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("Entry".equals(reader.getLocalName())) { + break; + } + } + } + + builder.fields(fields); + return builder.build(); + } + + /** + * Parse Fields element containing multiple Field elements. + * + * @param reader XMLStreamReader positioned at Fields start element + * @param fields List to add parsed fields to + */ + private void parseEntryFields(XMLStreamReader reader, List fields) + throws XMLStreamException { + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + if ("Field".equals(reader.getLocalName())) { + StreamedVulnerability.EntryField field = parseEntryField(reader); + if (field != null) { + fields.add(field); + } + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + } + } + } + + /** + * Parse a single Field element within Fields. + * + * @param reader XMLStreamReader positioned at Field start element + * @return Parsed EntryField or null + */ + private StreamedVulnerability.EntryField parseEntryField(XMLStreamReader reader) throws XMLStreamException { + StreamedVulnerability.EntryField.EntryFieldBuilder builder = + StreamedVulnerability.EntryField.builder(); + + // Get Field attributes + String type = reader.getAttributeValue(null, "type"); + String vulnTag = reader.getAttributeValue(null, "vulnTag"); + builder.type(type); + builder.vulnTag(vulnTag); + + int depth = 1; + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + switch (localName) { + case "Name": + builder.name(readElementText(reader)); + continue; + case "Value": + builder.value(readElementText(reader)); + continue; + } + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if ("Field".equals(reader.getLocalName())) { + break; + } + } + } + + return builder.build(); + } + + /** + * Parse ExternalID element. + * Structure: value + * + * @param reader XMLStreamReader positioned at ExternalID start element + * @param externalIdsMap Map to add parsed external ID to + */ + private void parseExternalID(XMLStreamReader reader, Map externalIdsMap) throws XMLStreamException { + String name = reader.getAttributeValue(null, "name"); + String value = readElementText(reader); + + if (name != null && value != null) { + externalIdsMap.put(name, value); + } + } + + /** + * Parse ReplacementDefinitions element. + * Structure: + * + * + * + * + * + * This replicates the logic from ReplacementParser for streaming parsing. + * + * @param reader XMLStreamReader positioned at ReplacementDefinitions start element + * @return Populated ReplacementData object or null if no definitions found + */ + private ReplacementData parseReplacementDefinitions(XMLStreamReader reader) + throws XMLStreamException { + + ReplacementData replacementData = + new ReplacementData(); + + int depth = 1; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("Def".equals(localName)) { + // Parse element: has key and value attributes, may have nested SourceLocation + String key = reader.getAttributeValue(null, "key"); + String value = reader.getAttributeValue(null, "value"); + + // Initialize location fields + String path = null; + String line = null; + String colStart = null; + String colEnd = null; + + // Check for nested SourceLocation element + int defDepth = 1; + while (reader.hasNext() && defDepth > 0) { + int defEvent = reader.next(); + + if (defEvent == XMLStreamConstants.START_ELEMENT) { + if ("SourceLocation".equals(reader.getLocalName())) { + // Extract SourceLocation attributes + path = reader.getAttributeValue(null, "path"); + line = reader.getAttributeValue(null, "line"); + colStart = reader.getAttributeValue(null, "colStart"); + colEnd = reader.getAttributeValue(null, "colEnd"); + } + defDepth++; + } else if (defEvent == XMLStreamConstants.END_ELEMENT) { + defDepth--; + } + } + + // Add replacement to data object + if (key != null) { + replacementData.addReplacement(key, value, path, line, colStart, colEnd); + logger.debug("Parsed Def: key={}, value={}, path={}", key, value, path); + } + continue; // Skip depth increment since we consumed the element + + } else if ("LocationDef".equals(localName)) { + // Parse element: has key and location attributes directly + String key = reader.getAttributeValue(null, "key"); + String path = reader.getAttributeValue(null, "path"); + String line = reader.getAttributeValue(null, "line"); + String colStart = reader.getAttributeValue(null, "colStart"); + String colEnd = reader.getAttributeValue(null, "colEnd"); + + if (key != null) { + Map attrs = new HashMap<>(); + attrs.put("path", path != null ? path : ""); + attrs.put("line", line != null ? line : "0"); + attrs.put("colStart", colStart != null ? colStart : "0"); + attrs.put("colEnd", colEnd != null ? colEnd : "0"); + + replacementData.addLocationReplacement(key, attrs); + logger.debug("Parsed LocationDef: key={}, path={}", key, path); + } + } + depth++; + + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + } + } + + logger.debug("Parsed ReplacementDefinitions: {} replacements, {} location replacements", + replacementData.getReplacements().size(), + replacementData.getLocationReplacements().size()); + + return replacementData; + } + + /** + * Helper to read element text safely. + */ + private String readElementText(XMLStreamReader reader) throws XMLStreamException { + if (reader.hasNext()) { + reader.next(); + if (reader.isCharacters()) { + return reader.getText(); + } + } + return null; + } + + /** + * Helper to parse integer safely. + */ + private Integer parseIntSafe(String value) { + if (value == null || value.isEmpty()) return null; + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return null; + } + } + + /** + * Helper to parse double safely. + */ + private Double parseDoubleSafe(String value) { + if (value == null || value.isEmpty()) return null; + try { + return Double.parseDouble(value); + } catch (NumberFormatException e) { + return null; + } + } + /** + * Process a StreamedVulnerability into a fully populated internal Vulnerability object. + * This method uses streaming-parsed metadata from FVDLMetadata instead of DOM-based processors. + * + * @param streamedVuln StreamedVulnerability object from streaming parser + * @param vulnFinalizer VulnFinalizer for calculating derived fields + * @return Fully processed Vulnerability object or null if creation fails + */ + public Vulnerability processVulnerability( + StreamedVulnerability streamedVuln, + VulnFinalizer vulnFinalizer) { + + if (streamedVuln == null || streamedVuln.getInstanceId() == null || streamedVuln.getClassId() == null) { + String instanceId = (streamedVuln != null) ? streamedVuln.getInstanceId() : "UNKNOWN"; + logger.warn("Skipping streamed vulnerability with instance ID [{}] due to missing critical data.", instanceId); + return null; + } + + // Create internal Vulnerability from StreamedVulnerability + Vulnerability vulnCustom = Vulnerability.builder() + .classID(streamedVuln.getClassId()) + .instanceID(streamedVuln.getInstanceId()) + .analyzerName(AnalyzerType.canonicalizeAnalyzerName(streamedVuln.getAnalyzerName())) + .type(streamedVuln.getType()) + .subType(streamedVuln.getSubType()) + .kingdom(streamedVuln.getKingdom()) + .defaultSeverity(streamedVuln.getDefaultSeverity()) + .instanceSeverity(streamedVuln.getInstanceSeverity()) + .confidence(streamedVuln.getConfidence()) + .analysis(streamedVuln.getShortDescription()) + .build(); + + // 1. Get the base metadata from streaming-parsed ruleMetadata in FVDLMetadata + Map finalMetadata = new HashMap<>(); + Map ruleMetadata = fvdlMetadata.getRuleMetadata().get(vulnCustom.getClassID()); + + if (ruleMetadata != null) { + finalMetadata.putAll(ruleMetadata); + } + + // 2. Apply any instance-specific metadata overrides from streaming parse + if (streamedVuln.getMetadata() != null) { + finalMetadata.putAll(streamedVuln.getMetadata()); + } + + // 3. Merge the final metadata into the vulnerability's knowledge map + vulnCustom.getKnowledge().putAll(finalMetadata); + + // 4. Populate high-level fields using merged data + vulnCustom.setAccuracy(XmlUtils.safeParseDouble(finalMetadata.get("Accuracy"), 0.0)); + vulnCustom.setImpact(XmlUtils.safeParseDouble(finalMetadata.get("Impact"), 0.0)); + vulnCustom.setProbability(XmlUtils.safeParseDouble(finalMetadata.get("Probability"), 0.0)); + + String audience = finalMetadata.getOrDefault("audience", ""); + vulnCustom.setAudience(audience); + vulnCustom.setFiletype(finalMetadata.getOrDefault("DefaultFile", "")); + + // Convert StreamedVulnerability Traces to StackTraces (hierarchical structure maintained) + List> stackTraces = new ArrayList<>(); + + if (streamedVuln.getTraces() != null && !streamedVuln.getTraces().isEmpty()) { + int totalNodes = streamedVuln.getTraces().stream() + .mapToInt(t -> t.getNodes() != null ? t.getNodes().size() : 0) + .sum(); + //logger.info("Traces count: {}, Total nodes: {}", streamedVuln.getTraces().size(), totalNodes); + + // Convert traces to stackTraces - already returns List> + stackTraces = convertTracesToStackTraces(streamedVuln.getTraces()); + } + + vulnCustom.setStackTrace(stackTraces); + + //Map uniqueFiles = new java.util.LinkedHashMap<>(); + if (!stackTraces.isEmpty()) { + List firstStackTrace = stackTraces.get(0); + List lastStackTrace = stackTraces.get(stackTraces.size() - 1); + vulnCustom.setFirstStackTrace(firstStackTrace); + vulnCustom.setSource(lastStackTrace.isEmpty() ? null : lastStackTrace.get(0)); + vulnCustom.setSink(lastStackTrace.isEmpty() ? null : lastStackTrace.get(lastStackTrace.size() - 1)); + vulnCustom.setLastStackTraceElement(lastStackTrace.isEmpty() ? null : lastStackTrace.get(lastStackTrace.size() - 1)); + vulnCustom.setLongestStackTrace(findLongestList(stackTraces)); + } + + aggregateFromTraces(vulnCustom); + + // Process DAST / Auxiliary data - replicate auxiliaryProcessor.process() and processRequestRelated() + processAuxiliaryAndRequestData(streamedVuln, vulnCustom); + + // Process descriptions from streaming-parsed descriptionCache in FVDLMetadata + ReplacementData replacementData = streamedVuln.getReplacementData(); + String[] descs = descriptionProcessor.processForVuln(vulnCustom, vulnCustom.getClassID(), replacementData, fvdlMetadata); + vulnCustom.setShortDescription(StringUtil.stripTags(descs[0], true)); + vulnCustom.setExplanation(StringUtil.stripTags(descs[1], true)); + + // Set projectName from first file path if not already set + // Since file content population is disabled, we derive it from stack trace + if ((vulnCustom.getProjectName() == null || vulnCustom.getProjectName().isEmpty()) + && !stackTraces.isEmpty() && !stackTraces.get(0).isEmpty()) { + com.fortify.cli.aviator.audit.model.StackTraceElement firstElement = stackTraces.get(0).get(0); + if (firstElement != null && firstElement.getFilename() != null) { + String firstFilePath = firstElement.getFilename(); + vulnCustom.setProjectName(initPackageName(firstFilePath)); + } + } + + // Finalize to calculate derived fields + vulnFinalizer.finalize(vulnCustom); + + return vulnCustom; + } + + /** + * Initialize package name from a file path. + * Extracts the first directory component as the project name. + * + * @param filePath File path to derive package from + * @return Derived package name or the full path if no separator found + */ + private String initPackageName(String filePath) { + if (filePath == null || filePath.trim().isEmpty()) { + return ""; + } + int separatorIndex = filePath.indexOf('/'); + return separatorIndex > 0 ? filePath.substring(0, separatorIndex) : filePath; + } + + /** + * Process auxiliary data and request-related fields from StreamedVulnerability. + * Replicates the functionality of auxiliaryProcessor.process() and processRequestRelated(). + * + * Now uses the fully parsed auxiliaryData and externalEntries from streaming parse. + * + * @param streamedVuln StreamedVulnerability from streaming parser + * @param vulnCustom Vulnerability object to populate + */ + private void processAuxiliaryAndRequestData( + StreamedVulnerability streamedVuln, + Vulnerability vulnCustom) { + + // Set auxiliaryData and externalEntries on vulnerability + // Convert StreamedVulnerability.ExternalEntry to Entry objects + List convertedEntries = convertExternalEntries(streamedVuln.getExternalEntries()); + vulnCustom.setAuxiliaryData(streamedVuln.getAuxiliaryData()); + vulnCustom.setExternalEntries(convertedEntries); + + // Process external IDs - add to knowledge map + if (streamedVuln.getExternalIds() != null) { + for (Map.Entry externalId : streamedVuln.getExternalIds().entrySet()) { + vulnCustom.getKnowledge().put("externalID." + externalId.getKey(), externalId.getValue()); + } + } + + // Extract DAST fields from auxiliary data and external entries + processRequestRelatedFromData(vulnCustom, streamedVuln.getAuxiliaryData(), convertedEntries); + } + + /** + * Convert StreamedVulnerability.ExternalEntry objects to Entry objects. + * Needed because Vulnerability class expects com.fortify.aviator.cli.fpr.models.Entry. + * + * @param streamedEntries List of StreamedVulnerability.ExternalEntry + * @return List of converted Entry objects + */ + private List convertExternalEntries( + List streamedEntries) { + + List result = new ArrayList<>(); + + if (streamedEntries == null) { + return result; + } + + for (StreamedVulnerability.ExternalEntry streamedEntry : streamedEntries) { + com.fortify.cli.aviator.fpr.model.Entry entry = new com.fortify.cli.aviator.fpr.model.Entry(); + entry.setUrl(streamedEntry.getUrl()); + + // Convert fields + List fields = new ArrayList<>(); + if (streamedEntry.getFields() != null) { + for (StreamedVulnerability.EntryField streamedField : streamedEntry.getFields()) { + com.fortify.cli.aviator.fpr.model.Entry.Field field = + new com.fortify.cli.aviator.fpr.model.Entry.Field(); + field.setName(streamedField.getName()); + field.setValue(streamedField.getValue()); + field.setType(streamedField.getType()); + field.setVulnTag(streamedField.getVulnTag()); + fields.add(field); + } + } + entry.setFields(fields); + + // Note: Function and SourceLocation objects are not fully converted here + // If needed, create JAXB objects from the streamed data + // For now, the essential fields (URL and Fields) are converted + + result.add(entry); + } + + return result; + } + + /** + * Process request-related fields from auxiliary data and external entries. + * This is a helper method that replicates FVDLProcessor.processRequestRelated() logic. + * + * To use this, you need to parse and store auxiliaryData and externalEntries during + * streaming parse phase. + * + * @param vulnCustom Vulnerability object to populate + * @param auxData List of auxiliary data maps + * @param externalEntries List of external entries + */ + private void processRequestRelatedFromData( + Vulnerability vulnCustom, + List> auxData, + List externalEntries) { + + // Process auxiliary data + if (auxData != null) { + for (Map aux : auxData) { + String contentType = aux.get("contentType"); + if (contentType != null) { + switch (contentType.toLowerCase()) { + case "requestheaders": + vulnCustom.setRequestHeaders(aux.values().stream() + .filter(v -> !v.equals(contentType)) + .collect(java.util.stream.Collectors.joining(","))); + break; + case "requestparameters": + vulnCustom.setRequestParameters(aux.values().stream() + .filter(v -> !v.equals(contentType)) + .collect(java.util.stream.Collectors.joining(","))); + break; + case "requestbody": + vulnCustom.setRequestBody(aux.get("value")); + break; + case "requestmethod": + vulnCustom.setRequestMethod(aux.get("value")); + break; + case "requestcookies": + vulnCustom.setRequestCookies(aux.get("value")); + break; + case "requesthttpversion": + vulnCustom.setRequestHttpVersion(aux.get("value")); + break; + case "attackpayload": + vulnCustom.setAttackPayload(aux.get("value")); + break; + case "attacktype": + vulnCustom.setAttackType(aux.get("value")); + break; + case "response": + vulnCustom.setResponse(aux.get("value")); + break; + case "trigger": + vulnCustom.setTrigger(aux.get("value")); + break; + case "vulnerableparameter": + vulnCustom.setVulnerableParameter(aux.get("value")); + break; + } + } + } + } + + // Process external entries + if (externalEntries != null) { + for (com.fortify.cli.aviator.fpr.model.Entry entry : externalEntries) { + if (entry.getUrl() != null && entry.getUrl().toLowerCase().contains("request")) { + for (com.fortify.cli.aviator.fpr.model.Entry.Field field : entry.getFields()) { + switch (field.getName().toLowerCase()) { + case "requestheaders": + vulnCustom.setRequestHeaders(field.getValue()); + break; + case "requestparameters": + vulnCustom.setRequestParameters(field.getValue()); + break; + case "requestbody": + vulnCustom.setRequestBody(field.getValue()); + break; + case "requestmethod": + vulnCustom.setRequestMethod(field.getValue()); + break; + case "requestcookies": + vulnCustom.setRequestCookies(field.getValue()); + break; + case "requesthttpversion": + vulnCustom.setRequestHttpVersion(field.getValue()); + break; + case "attackpayload": + vulnCustom.setAttackPayload(field.getValue()); + break; + case "attacktype": + vulnCustom.setAttackType(field.getValue()); + break; + case "response": + vulnCustom.setResponse(field.getValue()); + break; + case "trigger": + vulnCustom.setTrigger(field.getValue()); + break; + case "vulnerableparameter": + vulnCustom.setVulnerableParameter(field.getValue()); + break; + } + } + } + } + } + } + + /** + * Convert TraceNodes from StreamedVulnerability to StackTraceElements. + * Looks up taintFlags and knowledge from the Node pool using nodeId. + */ + /** + * Convert hierarchical Trace structure to StackTrace format. + * Each Trace becomes a List, maintaining the trace boundaries. + * + * @param traces List of Trace objects from StreamedVulnerability + * @return List of StackTrace lists (List>) + */ + private List> convertTracesToStackTraces( + List traces) { + logger.debug("[StreamingFVDLParser][convertTracesToStackTraces] Converting {} traces", + traces != null ? traces.size() : 0); + + List> allStackTraces = new ArrayList<>(); + + if (traces == null || traces.isEmpty()) { + return allStackTraces; + } + + // Process each trace separately to maintain boundaries + for (StreamedVulnerability.Trace trace : traces) { + List stackTrace = new ArrayList<>(); + + if (trace.getNodes() == null || trace.getNodes().isEmpty()) { + logger.warn("Trace has no nodes, skipping"); + continue; + } + + logger.debug("Processing trace with {} nodes", trace.getNodes().size()); + + // Convert each Node in the trace to a StackTraceElement + for (Node node : trace.getNodes()) { + // Convert node using extracted helper method (ensures consistency) + com.fortify.cli.aviator.audit.model.StackTraceElement element = + nodeToStackTraceElement(node); + + // NEW: Build innerStackTrace from Reason data + List innerStackTrace = + buildInnerStackTrace(node); + if (!innerStackTrace.isEmpty()) { + element.setInnerStackTrace(innerStackTrace); + logger.debug("Set innerStackTrace with {} elements for node {}", + innerStackTrace.size(), node.getId()); + } + + stackTrace.add(element); + } + + // Add this trace's stack trace to the collection + allStackTraces.add(stackTrace); + logger.debug("Converted trace to stackTrace with {} elements", stackTrace.size()); + } + + logger.debug("Converted {} traces to {} stackTraces", traces.size(), allStackTraces.size()); + return allStackTraces; + } + + /** + * Build innerStackTrace for a node based on its Reason element data. + * + * The Reason element can contain: + * - Inline Traces: Parsed during node parsing and stored in reasonInlineTraces + * - TraceRefs: References to TracePool traces, stored as IDs in reasonTraceRefs + * + * Both types are converted to StackTraceElements and combined into innerStackTrace. + * + * @param node Node with reasonTraceRefs and reasonInlineTraces from Reason parsing + * @return List of StackTraceElements for innerStackTrace (empty if no Reason data) + */ + private List buildInnerStackTrace( + Node node) { + + List innerTrace = new ArrayList<>(); + + if (node == null) { + return innerTrace; + } + + // Process inline traces from Reason element + if (node.getReasonInlineTraces() != null && !node.getReasonInlineTraces().isEmpty()) { + logger.debug("Building innerStackTrace from {} inline traces", node.getReasonInlineTraces().size()); + + for (com.fortify.cli.aviator.fpr.model.StreamedTrace inlineTrace : + node.getReasonInlineTraces()) { + List traceElements = + convertStreamedTraceToStackTrace(inlineTrace); + innerTrace.addAll(traceElements); + } + } + + // Process trace references from Reason element (resolve from TracePool) + if (node.getReasonTraceRefs() != null && !node.getReasonTraceRefs().isEmpty()) { + logger.debug("Building innerStackTrace from {} trace refs", node.getReasonTraceRefs().size()); + + for (String traceRefId : node.getReasonTraceRefs()) { + com.fortify.cli.aviator.fpr.model.StreamedTrace referencedTrace = + fvdlMetadata.getTracePool().get(traceRefId); + + if (referencedTrace != null) { + List traceElements = + convertStreamedTraceToStackTrace(referencedTrace); + innerTrace.addAll(traceElements); + } else { + logger.warn("TraceRef {} not found in TracePool for node {}", traceRefId, node.getId()); + } + } + } + + if (!innerTrace.isEmpty()) { + logger.debug("Built innerStackTrace with {} elements for node {}", innerTrace.size(), node.getId()); + } + + return innerTrace; + } + + /** + * Convert a StreamedTrace to StackTraceElements. + * + * This is similar to the main trace conversion logic but specifically for + * Reason traces (used in innerStackTrace building). + * + * @param trace StreamedTrace from TracePool or inline from Reason element + * @return List of StackTraceElements + */ + private List convertStreamedTraceToStackTrace( + com.fortify.cli.aviator.fpr.model.StreamedTrace trace) { + + List elements = new ArrayList<>(); + + if (trace == null || trace.getPrimary() == null || trace.getPrimary().getEntries() == null) { + return elements; + } + + logger.debug("Converting StreamedTrace with {} entries", trace.getPrimary().getEntries().size()); + + for (com.fortify.cli.aviator.fpr.model.StreamedTrace.Primary.Entry entry : + trace.getPrimary().getEntries()) { + + // CHANGED: Use Node object directly from Entry instead of NodePool lookup + // This preserves inline nodes (without IDs) that weren't added to NodePool + Node node = entry.getNode(); + + if (node != null) { + // Convert node to StackTraceElement using extracted helper + com.fortify.cli.aviator.audit.model.StackTraceElement element = + nodeToStackTraceElement(node); + elements.add(element); + } else { + // Fallback: Try NodePool lookup if Entry doesn't have node object (shouldn't happen) + String nodeId = entry.getNodeId(); + if (nodeId != null) { + node = fvdlMetadata.getNodePool().get(nodeId); + if (node != null) { + com.fortify.cli.aviator.audit.model.StackTraceElement element = + nodeToStackTraceElement(node); + elements.add(element); + } else { + logger.warn("Node {} referenced in trace not found (Entry has no node, NodePool lookup failed)", nodeId); + } + } else { + logger.warn("Entry has null node and null nodeId - skipping"); + } + } + } + + logger.debug("Converted StreamedTrace to {} StackTraceElements", elements.size()); + return elements; + } + + /** + * Convert a Node to a StackTraceElement. + * + * Extracted from convertTracesToStackTraces() for reuse in innerStackTrace building. + * This ensures consistent conversion logic for both main traces and inner traces. + * + * @param node Node to convert + * @return StackTraceElement + */ + private com.fortify.cli.aviator.audit.model.StackTraceElement nodeToStackTraceElement( + Node node) { + + // Get taintFlags as comma-separated string + String taintFlagsStr = ""; + if (node.getTaintFlags() != null && !node.getTaintFlags().isEmpty()) { + taintFlagsStr = String.join(", ", node.getTaintFlags()); + } + + com.fortify.cli.aviator.audit.model.Fragment fragment = + fileUtils.getFragmentFromFile( + this.fprHandle, + node.getFilePath(), + node.getLine(), + 5, // contextBefore - 5 lines before target line + 2 // contextAfter - 2 lines after target line + ); + + // Get code line from file + String codeLine = fileUtils.getLineFromFile(this.fprHandle, + node.getFilePath(), node.getLine()); + + // StackTraceElement constructor: (filename, line, code, nodeType, fragment, additionalInfo, taintflags) + com.fortify.cli.aviator.audit.model.StackTraceElement element = + new com.fortify.cli.aviator.audit.model.StackTraceElement( + node.getFilePath(), + node.getLine(), + codeLine, + node.getActionType() != null ? node.getActionType() : "", + fragment, + node.getAdditionalInfo() != null ? node.getAdditionalInfo() : "", + taintFlagsStr + ); + + // Set additional fields available via setters + if (node.getLabel() != null && !node.getLabel().isEmpty()) { + element.setReason(node.getLabel()); + } + + // Set knowledge map from Node + if (node.getKnowledge() != null) { + element.setKnowledge(node.getKnowledge()); + } + + return element; + } + /** + * Find the longest stack trace. + */ + private List findLongestList( + List> listOfLists) { + + if (listOfLists == null || listOfLists.isEmpty()) { + return new ArrayList<>(); + } + return listOfLists.stream() + .max(Comparator.comparingInt(this::getTotalTraceSize)) + .orElse(new ArrayList<>()); + } + + /** + * Calculate total trace size including nested elements. + */ + private int getTotalTraceSize(List trace) { + if (trace == null) { + return 0; + } + return trace.stream() + .mapToInt(this::countNodesRecursive) + .sum(); + } + + /** + * Recursively count nodes in a stack trace element. + */ + private int countNodesRecursive(com.fortify.cli.aviator.audit.model.StackTraceElement element) { + if (element == null) { + return 0; + } + int count = 1; + if (element.getInnerStackTrace() != null) { + for (com.fortify.cli.aviator.audit.model.StackTraceElement inner : element.getInnerStackTrace()) { + count += countNodesRecursive(inner); + } + } + return count; + } + + /** + * Aggregate taint flags and knowledge from all traces. + */ + private void aggregateFromTraces(Vulnerability vulnCustom) { + Set allTaintFlags = new HashSet<>(); + Map allKnowledge = new HashMap<>(); + + for (List trace : vulnCustom.getStackTrace()) { + for (com.fortify.cli.aviator.audit.model.StackTraceElement ste : trace) { + if (ste.getTaintflags() != null && !ste.getTaintflags().isEmpty()) { + allTaintFlags.addAll(Arrays.stream(ste.getTaintflags().split(",")) + .map(String::trim) + .collect(java.util.stream.Collectors.toSet())); + } + allKnowledge.putAll(ste.getKnowledge()); + + if (ste.getInnerStackTrace() != null) { + for (com.fortify.cli.aviator.audit.model.StackTraceElement inner : ste.getInnerStackTrace()) { + if (inner.getTaintflags() != null && !inner.getTaintflags().isEmpty()) { + allTaintFlags.addAll(Arrays.stream(inner.getTaintflags().split(",")) + .map(String::trim) + .collect(java.util.stream.Collectors.toSet())); + } + allKnowledge.putAll(inner.getKnowledge()); + } + } + } + } + vulnCustom.setTaintFlags(new ArrayList<>(allTaintFlags)); + vulnCustom.setKnowledge(allKnowledge); + } + +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/TraceParser.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/TraceParser.java new file mode 100644 index 00000000000..62147843544 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/TraceParser.java @@ -0,0 +1,184 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import static com.fortify.cli.aviator.fpr.processor.XmlParserUtils.getEventTypeName; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.fpr.model.*; + +/** + * Parses Trace elements from FVDL XML. + * Handles both UnifiedTracePool traces and inline traces within vulnerabilities. + */ +public class TraceParser { + private static final Logger logger = LoggerFactory.getLogger(TraceParser.class); + private NodeParser nodeParser; + private Map nodePool; + + public TraceParser(Map nodePool) { + this.nodePool = nodePool; + } + + public void setNodeParser(NodeParser nodeParser) { + this.nodeParser = nodeParser; + } + + /** + * Parse trace for UnifiedTracePool - returns StreamedTrace. + */ + public StreamedTrace parseStreamedTrace(XMLStreamReader reader, String traceId) throws XMLStreamException { + logger.debug("start parseTrace for traceId: {}", traceId); + + StreamedVulnerability.Trace trace = parseTrace(reader); + + if (trace == null || trace.getNodes() == null) { + logger.warn("parseTrace returned null or empty trace for traceId: {}", traceId); + return null; + } + + List entries = new ArrayList<>(); + + for (Node node : trace.getNodes()) { + StreamedTrace.Primary.Entry entry = + StreamedTrace.Primary.Entry.builder() + .nodeId(node.getId()) + .node(node) + .isDefault(node.isDetailsOnly()) + .build(); + entries.add(entry); + } + + StreamedTrace.Primary primary = + StreamedTrace.Primary.builder() + .entries(entries) + .build(); + + logger.debug("Completed parseTrace for traceId: {}, total entries: {}", traceId, entries.size()); + return StreamedTrace.builder() + .id(traceId) + .primary(primary) + .build(); + } + + /** + * Parse inline trace from Vulnerability. + */ + public StreamedVulnerability.Trace parseTrace(XMLStreamReader reader) throws XMLStreamException { + logger.debug("=== START parseTrace (vulnerability inline trace) ==="); + + List nodes = new ArrayList<>(); + boolean inPrimary = false; + + while (reader.hasNext()) { + int event = reader.next(); + + if((event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)) + logger.debug("[parseTrace] Event: {}, LocalName: {}", getEventTypeName(event), reader.getLocalName()); + + if (event == XMLStreamConstants.START_ELEMENT) { + String elementName = reader.getLocalName(); + + if ("Primary".equals(elementName)) { + inPrimary = true; + logger.debug("[parseTrace] Entered Primary section"); + + } else if (inPrimary && "Entry".equals(elementName)) { + logger.debug("[parseTrace] Parsing Entry element"); + parseTraceEntry(reader, nodes); + } + + } else if (event == XMLStreamConstants.END_ELEMENT) { + String elementName = reader.getLocalName(); + + if ("Primary".equals(elementName)) { + inPrimary = false; + + } else if ("Trace".equals(elementName)) { + logger.debug("=== END parseTrace - Parsed {} nodes ===", nodes.size()); + return StreamedVulnerability.Trace.builder() + .nodes(nodes) + .build(); + } + } + } + + logger.warn("=== END parseTrace (unexpected exit) - Parsed {} nodes ===", nodes.size()); + return StreamedVulnerability.Trace.builder() + .nodes(nodes) + .build(); + } + + /** + * Parse a single Entry element and add Node to the list. + */ + private void parseTraceEntry(XMLStreamReader reader, List nodes) throws XMLStreamException { + logger.debug("[parseTraceEntry] Start parsing Entry"); + + String nodeRefId = null; + Node inlineNode = null; + + while (reader.hasNext()) { + int event = reader.next(); + + if((event == XMLStreamConstants.START_ELEMENT || event == XMLStreamConstants.END_ELEMENT)) + logger.debug("[parseTraceEntry] Event: {}, LocalName: {}", getEventTypeName(event), reader.getLocalName()); + + if (event == XMLStreamConstants.START_ELEMENT) { + String localName = reader.getLocalName(); + + if ("NodeRef".equals(localName)) { + nodeRefId = reader.getAttributeValue(null, "id"); + logger.debug("[parseTraceEntry] Found NodeRef with id: {}", nodeRefId); + + } else if ("Node".equals(localName)) { + String nodeId = reader.getAttributeValue(null, "id"); + logger.debug("[parseTraceEntry] Found inline Node with id: {}", nodeId); + inlineNode = nodeParser.parseNode(reader, nodeId); + } + + } else if (event == XMLStreamConstants.END_ELEMENT) { + if ("Entry".equals(reader.getLocalName())) { + break; + } + } + } + + if (nodeRefId != null) { + Node node = nodePool.get(nodeRefId); + if (node != null) { + nodes.add(node); + logger.debug("[parseTraceEntry] Added Node for nodeId {}. Total nodes: {}", nodeRefId, nodes.size()); + } else { + logger.debug("[parseTraceEntry] NodeRef {} - Forward reference, creating placeholder", nodeRefId); + Node placeholder = nodeParser.createPlaceholderNode(nodeRefId); + nodes.add(placeholder); + } + } else if (inlineNode != null) { + nodes.add(inlineNode); + logger.debug("[parseTraceEntry] Added inline Node. Total nodes: {}", nodes.size()); + } else { + logger.warn("[parseTraceEntry] Entry has neither NodeRef nor inline Node - skipping"); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/XmlParserUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/XmlParserUtils.java new file mode 100644 index 00000000000..93cb4bbfd0f --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/processor/XmlParserUtils.java @@ -0,0 +1,171 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.processor; + +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class for common XML parsing operations. + * Handles element text reading, section skipping, and event type conversions. + */ +public class XmlParserUtils { + private static final Logger logger = LoggerFactory.getLogger(XmlParserUtils.class); + + /** + * Read element text content safely. + * Handles both simple text and CDATA content. + */ + public static String readElementText(XMLStreamReader reader) throws XMLStreamException { + StringBuilder text = new StringBuilder(); + while (reader.hasNext()) { + int event = reader.next(); + if (event == XMLStreamConstants.CHARACTERS || event == XMLStreamConstants.CDATA) { + text.append(reader.getText()); + } else if (event == XMLStreamConstants.END_ELEMENT) { + break; + } + } + return text.toString().trim(); + } + + /** + * Read element content with markup (preserves inner XML structure). + * Used for description fields that contain HTML/XML formatting. + */ + public static String readElementContentWithMarkup(XMLStreamReader reader, String elementName) throws XMLStreamException { + StringBuilder content = new StringBuilder(); + int depth = 1; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + + if (event == XMLStreamConstants.START_ELEMENT) { + depth++; + content.append("<").append(reader.getLocalName()); + for (int i = 0; i < reader.getAttributeCount(); i++) { + content.append(" ") + .append(reader.getAttributeLocalName(i)) + .append("=\"") + .append(reader.getAttributeValue(i)) + .append("\""); + } + content.append(">"); + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if (depth > 0) { + content.append(""); + } + } else if (event == XMLStreamConstants.CHARACTERS || event == XMLStreamConstants.CDATA) { + content.append(reader.getText()); + } + } + + return content.toString().trim(); + } + + /** + * Skip an entire XML section efficiently. + * Maintains proper depth tracking to skip nested elements. + */ + public static void skipSection(XMLStreamReader reader, String sectionName) throws XMLStreamException { + String startElement = reader.getLocalName(); + int depth = 1; + + while (reader.hasNext() && depth > 0) { + int event = reader.next(); + if (event == XMLStreamConstants.START_ELEMENT) { + depth++; + } else if (event == XMLStreamConstants.END_ELEMENT) { + depth--; + if (depth == 0 && startElement.equals(reader.getLocalName())) { + logger.trace("Skipped section: {}", sectionName); + return; + } + } + } + + logger.warn("Reached end of stream while skipping section: {}", sectionName); + } + + /** + * Get readable event type name for debugging. + */ + public static String getEventTypeName(int eventType) { + switch (eventType) { + case XMLStreamConstants.START_ELEMENT: return "START_ELEMENT"; + case XMLStreamConstants.END_ELEMENT: return "END_ELEMENT"; + case XMLStreamConstants.CHARACTERS: return "CHARACTERS"; + case XMLStreamConstants.CDATA: return "CDATA"; + case XMLStreamConstants.COMMENT: return "COMMENT"; + case XMLStreamConstants.SPACE: return "SPACE"; + case XMLStreamConstants.START_DOCUMENT: return "START_DOCUMENT"; + case XMLStreamConstants.END_DOCUMENT: return "END_DOCUMENT"; + case XMLStreamConstants.PROCESSING_INSTRUCTION: return "PROCESSING_INSTRUCTION"; + case XMLStreamConstants.ENTITY_REFERENCE: return "ENTITY_REFERENCE"; + case XMLStreamConstants.DTD: return "DTD"; + default: return "UNKNOWN(" + eventType + ")"; + } + } + + /** + * Safe integer parsing with null check. + */ + public static Integer parseIntSafe(String value) { + if (value == null || value.isEmpty()) { + return null; + } + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + return null; + } + } + + /** + * Safe double parsing with default value. + */ + public static double safeParseDouble(String value, double defaultValue) { + if (value == null || value.isEmpty()) { + return defaultValue; + } + try { + return Double.parseDouble(value); + } catch (NumberFormatException e) { + return defaultValue; + } + } + + /** + * Check if string is null or empty. + */ + public static boolean isNullOrEmpty(String str) { + return str == null || str.isEmpty(); + } + + /** + * Strip HTML/XML tags from text. + */ + public static String stripTags(String text, boolean preserveWhitespace) { + if (text == null) return ""; + String result = text.replaceAll("<[^>]+>", ""); + if (!preserveWhitespace) { + result = result.replaceAll("\\s+", " ").trim(); + } + return result; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java index d5c4c99219e..d275d50c3b7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/FileUtils.java @@ -28,7 +28,10 @@ import org.slf4j.LoggerFactory; import com.fortify.cli.aviator.audit.model.Fragment; +import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; +import com.fortify.cli.aviator.util.FileUtil; import com.fortify.cli.aviator.util.FprHandle; +import com.fortify.cli.aviator.util.LanguageCommentMapperUtil; public class FileUtils { private static final Logger logger = LoggerFactory.getLogger(FileUtils.class); @@ -138,4 +141,27 @@ public Optional getSourceFileContent(FprHandle fprHandle, String relativ return Optional.empty(); } } + + public String appendLineNumbers(String content, String fileName, int startLineNo) { + String fileExtension = FileUtil.getFileExtension(fileName); + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage(fileExtension); + String commentSymbol = LanguageCommentMapperUtil.getProgrammingLanguageComment(language); + if(commentSymbol.equals("Unknown")) { + logger.warn("No Comment symbol is there so line numbers not appended"); + return content; + } + StringBuilder result = new StringBuilder(); + String[] lines = content.split("\\R", -1); // keep trailing empty lines + int endLineNo = startLineNo + lines.length; + for (int i = startLineNo; i < endLineNo; i++) { + result.append(lines[i-startLineNo]).append(" ").append(commentSymbol).append(" L").append(i + 1); + if(commentSymbol.equals(""); + else if(commentSymbol.equals("<%--")) + result.append(" --%>"); + if(i!=endLineNo-1) + result.append(System.lineSeparator()); + } + return result.toString(); + } } diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java new file mode 100644 index 00000000000..bbc0688ec73 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/utils/SourceCodeEnricher.java @@ -0,0 +1,154 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.fpr.utils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.aviator.audit.model.File; +import com.fortify.cli.aviator.audit.model.StackTraceElement; +import com.fortify.cli.aviator.util.FprHandle; +import com.fortify.cli.aviator.util.StringUtil; + + +/** + * Utility class for enriching stack traces with source code files. + * Extracts unique files from stack traces and loads their content from the FPR extraction directory. + * + * This class provides the same logic as FVDLProcessor.processStackTraceElements() but as a + * reusable, injectable component that can be used in different contexts (e.g., just-in-time + * enrichment before sending to LLM). + */ +public class SourceCodeEnricher { + private static final Logger logger = LoggerFactory.getLogger(SourceCodeEnricher.class); + + /*private final Path extractedPath; + private final Map sourceFileMap;*/ + private final FprHandle fprHandle; + private final FileUtils fileUtils; + + /** + * Creates a new SourceCodeEnricher with the required dependencies. + * @param fprHandle Utility for file operations (line numbering, line counting) + */ + /*public SourceCodeEnricher(Path extractedPath, Map sourceFileMap, FileUtils fileUtils) { + this.extractedPath = extractedPath; + this.sourceFileMap = sourceFileMap; + this.fileUtils = fileUtils; + }*/ + + public SourceCodeEnricher(FprHandle fprHandle){ + this.fprHandle = fprHandle; + this.fileUtils = new FileUtils(); + } + + /** + * Enriches stack traces with source code files. + * Extracts unique files from all stack traces (including inner traces) and loads their content. + * + * This method processes: + * - All stack traces in the list + * - All elements in each stack trace + * - All inner stack traces recursively + * + * Files are deduplicated - each unique filename is only loaded once. + * + * @param stackTraces List of stack traces to process + * @return Map of filename → File objects with content loaded + */ + public Map enrichWithSourceCode(List> stackTraces) { + Map uniqueFiles = new HashMap<>(); + + if (stackTraces == null || stackTraces.isEmpty()) { + logger.debug("No stack traces to enrich"); + return uniqueFiles; + } + + processStackTraces(stackTraces, uniqueFiles); + + logger.debug("Enriched {} unique source files from {} stack traces", + uniqueFiles.size(), stackTraces.size()); + + return uniqueFiles; + } + + /** + * Processes all stack traces to extract and load unique source files. + * Replicates FVDLProcessor.processStackTraceElements() logic. + */ + private void processStackTraces(List> stackTraces, Map uniqueFiles) { + for (List stackTrace : stackTraces) { + if (stackTrace == null) continue; + + for (StackTraceElement element : stackTrace) { + processFileForElement(element, uniqueFiles); + + // Process inner stack traces recursively + if (element.getInnerStackTrace() != null) { + for (StackTraceElement innerElement : element.getInnerStackTrace()) { + processFileForElement(innerElement, uniqueFiles); + } + } + } + } + } + + /** + * Processes a single stack trace element to extract and load its source file. + * Replicates FVDLProcessor.processFileForElement() logic. + * + * @param element The stack trace element to process + * @param uniqueFiles Map to store loaded files (deduplicated by filename) + */ + private void processFileForElement(StackTraceElement element, Map uniqueFiles) { + if (element == null) return; + + String filename = element.getFilename(); + if (!StringUtil.isEmpty(filename) && fprHandle.getSourceFileMap().containsKey(filename) && !uniqueFiles.containsKey(filename)) { + String internalPath = fprHandle.getSourceFileMap().get(filename); + if (internalPath == null) { return; } // Should not happen due to containsKey check, but safe. + + Path actualSourcePath = fprHandle.getPath("/" + internalPath); + + File file = new File(); + file.setName(filename); + file.setSegment(false); + file.setStartLine(1); + + try { + if (Files.exists(actualSourcePath)) { + byte[] encodedBytes = Files.readAllBytes(actualSourcePath); + file.setContent(new String(encodedBytes)); + file.setEndLine(fileUtils.countLines(actualSourcePath)); + } else { + // This warning is now more accurate. + logger.warn("Source file not found at internal path: {}. This may indicate a corrupt FPR.", actualSourcePath); + file.setContent(""); + file.setEndLine(0); + } + } catch (IOException e) { + logger.warn("Error processing file: {}", filename, e); + file.setContent(""); + file.setEndLine(0); + } + uniqueFiles.put(filename, file); + } + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java index d5302e6e423..772a74461e7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorGrpcClient.java @@ -44,6 +44,7 @@ import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FprHandle; import com.fortify.grpc.token.DeleteTokenRequest; import com.fortify.grpc.token.DeleteTokenResponse; import com.fortify.grpc.token.ListTokensByDeveloperRequest; @@ -109,8 +110,8 @@ public AviatorGrpcClient(ManagedChannel channel, long defaultTimeoutSeconds, IAv this(channel, defaultTimeoutSeconds, logger, 30); } - public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token) { - AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, pingIntervalSeconds, defaultTimeoutSeconds); + public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token, FprHandle fprHandle) { + AviatorStreamProcessor processor = new AviatorStreamProcessor(this, logger, asyncStub, processingExecutor, pingScheduler, pingIntervalSeconds, defaultTimeoutSeconds, fprHandle); CompletableFuture> future = processor.processBatchRequests(requests, projectName, FPRBuildId, SSCApplicationName, SSCApplicationVersion, token); future.whenComplete((res, th) -> processor.close()); return future.exceptionally(ex -> { @@ -231,4 +232,4 @@ public List listEntitlements(String tenantName, String signature, S ListEntitlementsByTenantResponse response = GrpcUtil.executeGrpcCall(entitlementServiceBlockingStub, EntitlementServiceGrpc.EntitlementServiceBlockingStub::listEntitlementsByTenant, request, Constants.OP_LIST_ENTITLEMENTS); return response.getEntitlementsList(); } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java index e4a480579ff..9b2c4ceafa7 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/AviatorStreamProcessor.java @@ -40,18 +40,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fortify.aviator.grpc.AuditRequest; -import com.fortify.aviator.grpc.AuditorResponse; -import com.fortify.aviator.grpc.AuditorServiceGrpc; -import com.fortify.aviator.grpc.PingRequest; -import com.fortify.aviator.grpc.StreamInitRequest; -import com.fortify.aviator.grpc.UserPromptRequest; +import com.fortify.aviator.grpc.*; import com.fortify.cli.aviator._common.exception.AviatorSimpleException; import com.fortify.cli.aviator._common.exception.AviatorTechnicalException; import com.fortify.cli.aviator.audit.model.AuditResponse; import com.fortify.cli.aviator.audit.model.UserPrompt; import com.fortify.cli.aviator.config.IAviatorLogger; +import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; +import com.fortify.cli.aviator.util.FileUtil; +import com.fortify.cli.aviator.util.FprHandle; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -91,8 +90,9 @@ class AviatorStreamProcessor implements AutoCloseable { private CountDownLatch streamLatch; private volatile Future processingTask; private final Object retryLock = new Object(); + private final FprHandle fprHandle; - public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, long defaultTimeoutSeconds) { + public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, AuditorServiceGrpc.AuditorServiceStub asyncStub, ExecutorService processingExecutor, ScheduledExecutorService pingScheduler, long pingIntervalSeconds, long defaultTimeoutSeconds, FprHandle fprHandle) { this.client = client; this.logger = logger; this.asyncStub = asyncStub; @@ -100,6 +100,7 @@ public AviatorStreamProcessor(AviatorGrpcClient client, IAviatorLogger logger, A this.pingScheduler = pingScheduler; this.pingIntervalSeconds = pingIntervalSeconds; this.defaultTimeoutSeconds = defaultTimeoutSeconds; + this.fprHandle = fprHandle; } public CompletableFuture> processBatchRequests(Queue requests, String projectName, String FPRBuildId, String SSCApplicationName, String SSCApplicationVersion, String token) { @@ -583,6 +584,7 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque } wrapper = processingQueue.poll(); + if (wrapper == null) { requestSemaphore.release(); continue; @@ -590,6 +592,19 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque String instanceId = wrapper.userPrompt.getIssueData().getInstanceID(); + // Lazy Loading of source code files for individual issue + SourceCodeEnricher sourceCodeEnricher = new SourceCodeEnricher(fprHandle); + + Map enrichedFiles = + sourceCodeEnricher.enrichWithSourceCode(wrapper.userPrompt.getStackTrace()); + List sourceCodeFiles = new ArrayList<>(enrichedFiles.values()); + + wrapper.userPrompt.getFiles().addAll(sourceCodeFiles); + wrapper.userPrompt.getProgrammingLanguages().addAll(programmingLanguages(sourceCodeFiles)); + + logger.info("Size of files {}", wrapper.userPrompt.getFiles().size()); + logger.info("Size of programming language {}", wrapper.userPrompt.getProgrammingLanguages().size()); + if (currentStreamState.processedIssueIds.contains(instanceId)) { requestSemaphore.release(); continue; @@ -663,6 +678,20 @@ private void processRequestQueue(int totalRequests, AtomicInteger processedReque processingQueue.size(), processedRequests.get(), totalRequests, outstandingRequests.get()); } + private Set programmingLanguages(List sourceCodeFiles){ + Set programmingLanguages = new HashSet<>(); + for (com.fortify.cli.aviator.audit.model.File file : sourceCodeFiles) { + String fileExtension = FileUtil.getFileExtension(file.getName()); + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage(fileExtension); + if (language != null) { + programmingLanguages.add(language); + } + } + return programmingLanguages; + } + + + private void handleServerBusy(String requestId, int totalRequests, AtomicInteger processedRequests, Map responses, CompletableFuture> resultFuture, CountDownLatch streamLatch) { RequestWrapper wrapperToRetry = inflightRequests.remove(requestId); if (wrapperToRetry == null) { @@ -844,4 +873,4 @@ public void close() { } } } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java index 3e70604ffd0..79c2bd579a0 100644 --- a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/grpc/GrpcUtil.java @@ -12,8 +12,7 @@ */ package com.fortify.cli.aviator.grpc; -import java.util.ArrayList; -import java.util.List; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -34,7 +33,10 @@ import com.fortify.cli.aviator.audit.model.Change; import com.fortify.cli.aviator.audit.model.StackTraceElement; import com.fortify.cli.aviator.audit.model.UserPrompt; +import com.fortify.cli.aviator.fpr.utils.SourceCodeEnricher; import com.fortify.cli.aviator.util.Constants; +import com.fortify.cli.aviator.util.FileTypeLanguageMapperUtil; +import com.fortify.cli.aviator.util.FileUtil; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -42,6 +44,7 @@ class GrpcUtil { private static final Logger LOG = LoggerFactory.getLogger(GrpcUtil.class); + SourceCodeEnricher sourceCodeEnricher; @FunctionalInterface interface GrpcCall { @@ -135,15 +138,20 @@ static AuditRequest convertToAuditRequest(UserPrompt userPrompt, String streamId if (userPrompt.getLongestStackTrace() != null) { builder.addAllLongestStackTrace(userPrompt.getLongestStackTrace().stream().map(GrpcUtil::convertToStackTraceElement).collect(Collectors.toList())); } + if (userPrompt.getFiles() != null) { builder.addAllFiles(userPrompt.getFiles().stream().map(file -> File.newBuilder().setName(file.getName() == null ? "" : file.getName()).setContent(file.getContent() == null ? "" : file.getContent()).setSegment(file.isSegment()).setStartLine(file.getStartLine()).setEndLine(file.getEndLine()).build()).collect(Collectors.toList())); } if (userPrompt.getLastStackTraceElement() != null) { builder.setLastStackTraceElement(convertToStackTraceElement(userPrompt.getLastStackTraceElement())); } - if (userPrompt.getProgrammingLanguages() != null) { + + //We are setting list of files later for lazy loading that's why programming language will be unavailable + if(userPrompt.getFiles()!=null) + builder.addAllProgrammingLanguages(programmingLanguages(userPrompt.getFiles())); + /*if (userPrompt.getProgrammingLanguages() != null) { builder.addAllProgrammingLanguages(userPrompt.getProgrammingLanguages()); - } + }*/ builder.setFileExtension(userPrompt.getFileExtension() == null ? "" : userPrompt.getFileExtension()); builder.setLanguage(userPrompt.getLanguage() == null ? "" : userPrompt.getLanguage()); builder.setCategory(userPrompt.getCategory() == null ? "" : userPrompt.getCategory()); @@ -160,6 +168,18 @@ static AuditRequest convertToAuditRequest(UserPrompt userPrompt, String streamId return builder.build(); } + private static Set programmingLanguages(List sourceCodeFiles){ + Set programmingLanguages = new HashSet<>(); + for (com.fortify.cli.aviator.audit.model.File file : sourceCodeFiles) { + String fileExtension = FileUtil.getFileExtension(file.getName()); + String language = FileTypeLanguageMapperUtil.getProgrammingLanguage(fileExtension); + if (language != null) { + programmingLanguages.add(language); + } + } + return programmingLanguages; + } + private static com.fortify.aviator.grpc.StackTraceElement convertToStackTraceElement(StackTraceElement element) { if (element == null) return com.fortify.aviator.grpc.StackTraceElement.getDefaultInstance(); @@ -201,4 +221,4 @@ static AuditResponse convertToAuditResponse(AuditorResponse response) { auditResponse.setSystemPrompt(response.getSystemPrompt()); return auditResponse; } -} \ No newline at end of file +} diff --git a/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/LanguageCommentMapperUtil.java b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/LanguageCommentMapperUtil.java new file mode 100644 index 00000000000..e8be89d02d3 --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/util/LanguageCommentMapperUtil.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.aviator.util; + +import com.fortify.cli.aviator.config.LanguagesCommentConfig; + +public class LanguageCommentMapperUtil { + private static LanguagesCommentConfig commentsConfig; + + public static void initializeConfig(LanguagesCommentConfig loadedCommentsConfig) { + commentsConfig = loadedCommentsConfig; + } + + public static String getProgrammingLanguageComment(String language) { + if(StringUtil.isEmpty(language)){ + return "Unknown"; + } + + return commentsConfig != null + ? commentsConfig.getCommentForLanguage(language) + : "Unknown"; + } +} diff --git a/fcli-core/fcli-aviator-common/src/main/resources/languages_comment_config.yaml b/fcli-core/fcli-aviator-common/src/main/resources/languages_comment_config.yaml new file mode 100644 index 00000000000..9e40210db7d --- /dev/null +++ b/fcli-core/fcli-aviator-common/src/main/resources/languages_comment_config.yaml @@ -0,0 +1,50 @@ +lineCommentSymbols: + JAVA: "//" + JSP: "<%--" + TLD: "