Implement streaming FVDL processing with memory tracking to optimize …

fcli-core/fcli-aviator-common/build.gradle.kts

@@ -14,10 +14,15 @@ tasks.withType<JavaCompile>().configureEach { dependsOn("generateProto") }
 dependencies {
     implementation(project(":fcli-core:fcli-common"))
     implementation("org.yaml:snakeyaml:2.3")
-    implementation("jakarta.xml.bind:jakarta.xml.bind-api:2.3.3")
-    implementation("org.glassfish.jaxb:jaxb-runtime:2.3.3")
+
+    // JAXB for XML object marshalling (used in FVDLProcessor legacy parser)
+    implementation("jakarta.xml.bind:jakarta.xml.bind-api:3.0.1")
+    implementation("org.glassfish.jaxb:jaxb-runtime:3.0.2")
     implementation("com.sun.activation:jakarta.activation:2.0.1")
-    implementation("jakarta.xml.ws:jakarta.xml.ws-api:3.0.1")
+
+    // Note: StAX (javax.xml.stream) uses Woodstox 7.1.1 via jackson-dataformat-xml
+    // from fcli-common (needed for XML output). No explicit dependency required.
+
     implementation("com.auth0:java-jwt:4.5.0")
     implementation("io.grpc:grpc-netty-shaded:1.76.0")
     implementation("io.grpc:grpc-protobuf:1.76.0")

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/AuditFPR.java

@@ -37,7 +37,7 @@
 import com.fortify.cli.aviator.fpr.model.AuditIssue;
 import com.fortify.cli.aviator.fpr.model.FPRInfo;
 import com.fortify.cli.aviator.fpr.processor.AuditProcessor;
-import com.fortify.cli.aviator.fpr.processor.FVDLProcessor;
+import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor;
 import com.fortify.cli.aviator.util.FprHandle;
 import com.fortify.cli.aviator.util.ResourceUtil;
 
@@ -65,28 +65,29 @@ public static FPRAuditResult auditFPR(AuditFprOptions options)
         Map<String, AuditResponse> auditResponses = new ConcurrentHashMap<>();
         AuditOutcome auditOutcome = performAviatorAudit(
                 parsedData, options.getLogger(), options.getToken(), options.getAppVersion(), options.getUrl(), options.getSscAppName(), options.getSscAppVersion(),
-                auditResponses, filterSelection
+                auditResponses, filterSelection, options.getFprHandle()
         );
 
         // --- STAGE 4: FINALIZATION ---
         return finalizeFprAudit(
                 auditOutcome, auditResponses, parsedData.auditProcessor,
-                tagMappingConfig, parsedData.fprInfo, parsedData.fvdlProcessor
+                tagMappingConfig, parsedData.fprInfo
         );
     }
 
     private static ParsedFprData prepareAndParseFpr(FprHandle fprHandle) {
         try {
             // Processors now take the FprHandle directly, no more extracted path
             AuditProcessor auditProcessor = new AuditProcessor(fprHandle);
-            FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle);
+            //FVDLProcessor fvdlProcessor = new FVDLProcessor(fprHandle);
+            StreamingFVDLProcessor streamingFVDLProcessor = new StreamingFVDLProcessor(fprHandle);
 
             Map<String, AuditIssue> auditIssueMap = auditProcessor.processAuditXML();
             FPRProcessor fprProcessor = new FPRProcessor(fprHandle, auditIssueMap, auditProcessor);
-            List<Vulnerability> vulnerabilities = fprProcessor.process(fvdlProcessor);
+            List<Vulnerability> vulnerabilities = fprProcessor.process(streamingFVDLProcessor);
             FPRInfo fprInfo = fprProcessor.getFprInfo();
 
-            return new ParsedFprData(auditIssueMap, vulnerabilities, fprInfo, auditProcessor, fvdlProcessor);
+            return new ParsedFprData(auditIssueMap, vulnerabilities, fprInfo, auditProcessor, streamingFVDLProcessor);
         } catch (Exception e) {
             LOG.error("A critical error occurred during FPR processing.", e);
             throw new AviatorTechnicalException("Failed to process FPR contents.", e);
@@ -106,21 +107,21 @@ private static TagMappingConfig loadTagMappingConfig(String tagMappingFilePath)
     private static AuditOutcome performAviatorAudit(
             ParsedFprData parsedData, IAviatorLogger logger,
             String token, String appVersion, String url, String sscAppName, String sscAppVersion,
-            Map<String, AuditResponse> auditResponsesToFill, FilterSelection filterSelection) {
+            Map<String, AuditResponse> auditResponsesToFill, FilterSelection filterSelection, FprHandle fprHandle) {
 
         IssueAuditor issueAuditor = new IssueAuditor(
                 parsedData.vulnerabilities, parsedData.auditProcessor, parsedData.auditIssueMap,
                 parsedData.fprInfo, sscAppName, sscAppVersion, filterSelection, logger
         );
         return issueAuditor.performAudit(
-                auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url
+                auditResponsesToFill, token, appVersion, parsedData.fprInfo.getBuildId(), url, fprHandle
         );
     }
 
     private static FPRAuditResult finalizeFprAudit(
             AuditOutcome auditOutcome, Map<String, AuditResponse> auditResponses,
             AuditProcessor auditProcessor, TagMappingConfig tagMappingConfig,
-            FPRInfo fprInfo, FVDLProcessor fvdlProcessor) {
+            FPRInfo fprInfo) {
 
         int totalIssuesToAudit = auditOutcome.getTotalIssuesToAudit();
         if (auditResponses.isEmpty()) {
@@ -160,10 +161,10 @@ private static FPRAuditResult finalizeFprAudit(
 
         File updatedFile = null;
         if (issuesSuccessfullyAudited > 0) {
-            updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, fprInfo, fvdlProcessor);
+            updatedFile = auditProcessor.updateAndSaveAuditAndRemediationsXml(auditResponses, tagMappingConfig, fprInfo);
         }
 
         LOG.info("FPR audit process completed with status: {}", status);
         return new FPRAuditResult(updatedFile, status, message, (int) issuesSuccessfullyAudited, totalIssuesToAudit);
     }
-}
+}

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/IssueAuditor.java

@@ -51,6 +51,7 @@
 import com.fortify.cli.aviator.grpc.AviatorGrpcClient;
 import com.fortify.cli.aviator.grpc.AviatorGrpcClientHelper;
 import com.fortify.cli.aviator.util.Constants;
+import com.fortify.cli.aviator.util.FprHandle;
 import com.fortify.cli.aviator.util.StringUtil;
 
 
@@ -141,7 +142,7 @@ private TagDefinition resolveHumanAuditStatus() {
     }
 
     public AuditOutcome performAudit(Map<String, AuditResponse> auditResponses, String token,
-                                    String projectName, String projectBuildId, String url) {
+                                     String projectName, String projectBuildId, String url, FprHandle fprHandle) {
         projectName = StringUtil.isEmpty(projectName) ? projectBuildId : projectName;
         logger.progress("Starting audit for project: %s", projectName);
 
@@ -154,7 +155,7 @@ public AuditOutcome performAudit(Map<String, AuditResponse> auditResponses, Stri
         } else {
             try (AviatorGrpcClient client = AviatorGrpcClientHelper.createClient(url, logger, DEFAULT_PING_INTERVAL_SECONDS)) {
                 CompletableFuture<Map<String, AuditResponse>> future =
-                        client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token);
+                        client.processBatchRequests(promptsToAudit, projectName, fprInfo.getBuildId(), SSCApplicationName, SSCApplicationVersion, token, fprHandle);
                 Map<String, AuditResponse> responses = future.get(500, TimeUnit.MINUTES);
                 responses.forEach((requestId, response) -> auditResponses.put(response.getIssueId(), response));
                 logger.progress("Audit completed");

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/audit/model/ParsedFprData.java

@@ -19,7 +19,7 @@
 import com.fortify.cli.aviator.fpr.model.AuditIssue;
 import com.fortify.cli.aviator.fpr.model.FPRInfo;
 import com.fortify.cli.aviator.fpr.processor.AuditProcessor;
-import com.fortify.cli.aviator.fpr.processor.FVDLProcessor;
+import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor;
 
 /**
  * A data-holding class that represents the complete, parsed contents of an FPR file.
@@ -31,13 +31,14 @@ public final class ParsedFprData {
     public final List<Vulnerability> vulnerabilities;
     public final FPRInfo fprInfo;
     public final AuditProcessor auditProcessor;
-    public final FVDLProcessor fvdlProcessor;
+    //public final FVDLProcessor fvdlProcessor;
+    public final StreamingFVDLProcessor streamingFVDLProcessor;
 
-    public ParsedFprData(Map<String, AuditIssue> auditIssueMap, List<Vulnerability> vulnerabilities, FPRInfo fprInfo, AuditProcessor auditProcessor, FVDLProcessor fvdlProcessor) {
+    public ParsedFprData(Map<String, AuditIssue> auditIssueMap, List<Vulnerability> vulnerabilities, FPRInfo fprInfo, AuditProcessor auditProcessor, StreamingFVDLProcessor streamingFvdlProcessor) {
         this.auditIssueMap = auditIssueMap;
         this.vulnerabilities = vulnerabilities;
         this.fprInfo = fprInfo;
         this.auditProcessor = auditProcessor;
-        this.fvdlProcessor = fvdlProcessor;
+        this.streamingFVDLProcessor = streamingFvdlProcessor;
     }
-}
+}

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/config/LanguagesCommentConfig.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2021-2026 Open Text.
+ *
+ * The only warranties for products and services of Open Text
+ * and its affiliates and licensors ("Open Text") are as may
+ * be set forth in the express warranty statements accompanying
+ * such products and services. Nothing herein should be construed
+ * as constituting an additional warranty. Open Text shall not be
+ * liable for technical or editorial errors or omissions contained
+ * herein. The information contained herein is subject to change
+ * without notice.
+ */
+package com.fortify.cli.aviator.config;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.fortify.cli.aviator.util.StringUtil;
+
+public class LanguagesCommentConfig {
+    private Map<String, String> lineCommentSymbols = new HashMap<>();
+
+    public void setLineCommentSymbols(Map<String, String> comments) {
+        this.lineCommentSymbols = comments != null ? new HashMap<>(comments) : new HashMap<>();
+    }
+
+    public Map<String, String> getLineCommentSymbols() {
+        return new HashMap<>(lineCommentSymbols);
+    }
+
+    public String getCommentForLanguage(String language) {
+        if (StringUtil.isEmpty(language)) {
+            return "Unknown";
+        }
+
+        return lineCommentSymbols.getOrDefault(language, "Unknown");
+    }
+}

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/FPRProcessor.java

@@ -15,6 +15,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.zip.ZipFile;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -25,8 +26,8 @@
 import com.fortify.cli.aviator.fpr.model.AuditIssue;
 import com.fortify.cli.aviator.fpr.model.FPRInfo;
 import com.fortify.cli.aviator.fpr.processor.AuditProcessor;
-import com.fortify.cli.aviator.fpr.processor.FVDLProcessor;
 import com.fortify.cli.aviator.fpr.processor.FilterTemplateParser;
+import com.fortify.cli.aviator.fpr.processor.StreamingFVDLProcessor;
 import com.fortify.cli.aviator.util.FprHandle;
 
 import lombok.Getter;
@@ -50,10 +51,10 @@ public FPRProcessor(FprHandle fprHandle, Map<String, AuditIssue> auditIssueMap,
     /**
      * Processes the main components of the FPR.
      *
-     * @param fvdlProcessor The processor for handling the audit.fvdl file.
+     * @param streamingFVDLProcessor The processor for handling the audit.fvdl file.
      * @return A list of all vulnerabilities found in the FVDL.
      */
-    public List<Vulnerability> process(FVDLProcessor fvdlProcessor) {
+    public List<Vulnerability> process(StreamingFVDLProcessor streamingFVDLProcessor) {
         logger.info("FPR Processing started");
     try{
         this.fprInfo = new FPRInfo(this.fprHandle);
@@ -83,7 +84,13 @@ public List<Vulnerability> process(FVDLProcessor fvdlProcessor) {
 
         logger.debug("Audit.xml Issues found: {}", auditIssueMap.size());
 
-        List<Vulnerability> vulnerabilities = fvdlProcessor.processXML();
+        try (ZipFile zipFile = new ZipFile(fprHandle.getFprPath().toFile())) {
+            streamingFVDLProcessor.parse(zipFile, "audit.fvdl");
+        }
+
+        //List<Vulnerability> vulnerabilities = fvdlProcessor.processXML();
+
+        List<Vulnerability> vulnerabilities = streamingFVDLProcessor.getVulnerabilities();
         logger.info("Parsed {} vulnerabilities from FVDL.", vulnerabilities.size());
 
         return vulnerabilities;
@@ -94,4 +101,4 @@ public List<Vulnerability> process(FVDLProcessor fvdlProcessor) {
         throw new AviatorTechnicalException("Unexpected error during FPR processing.", e);
         }
     }
-}
+}

fcli-core/fcli-aviator-common/src/main/java/com/fortify/cli/aviator/fpr/model/FPRInfo.java

@@ -53,13 +53,127 @@ public class FPRInfo {
     public FPRInfo(FprHandle fprHandle) {
         FPRName = String.valueOf(fprHandle.getFprPath().getFileName());
         try {
-            extractInfoFromAuditFvdl(fprHandle);
+            extractInfoFromAuditFvdlStreaming(fprHandle);
         } catch (Exception e) {
             // It's better to wrap this in a specific runtime exception
             throw new RuntimeException("Failed to extract info from audit.fvdl", e);
         }
     }
 
+    /**
+     * Extract FPR metadata from audit.fvdl using streaming XML parsing (StAX).
+     * More memory-efficient than DOM parsing for large files.
+     *
+     * Extracts:
+     * - UUID
+     * - Build information (BuildID, SourceBasePath, NumberFiles, ScanTime)
+     *
+     * @param fprHandle for getting path
+     * @throws Exception if parsing fails
+     */
+    private void extractInfoFromAuditFvdlStreaming(FprHandle fprHandle) throws Exception {
+        Path auditPath = fprHandle.getPath("/audit.fvdl");
+
+        if (!Files.exists(auditPath)) {
+            throw new IllegalStateException("audit.fvdl not found in FPR: " + fprHandle.getFprPath());
+        }
+
+        javax.xml.stream.XMLInputFactory factory = javax.xml.stream.XMLInputFactory.newInstance();
+        // Security: Disable external entity processing
+        factory.setProperty(javax.xml.stream.XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+        factory.setProperty(javax.xml.stream.XMLInputFactory.SUPPORT_DTD, false);
+
+        try (java.io.InputStream inputStream = Files.newInputStream(auditPath)) {
+            javax.xml.stream.XMLStreamReader reader = factory.createXMLStreamReader(inputStream);
+
+            boolean inBuild = false;
+            String currentElement = null;
+
+            while (reader.hasNext()) {
+                int event = reader.next();
+
+                if (event == javax.xml.stream.XMLStreamConstants.START_ELEMENT) {
+                    String localName = reader.getLocalName();
+
+                    if ("UUID".equals(localName)) {
+                        // Extract UUID text content
+                        this.uuid = readElementText(reader);
+
+                    } else if ("Build".equals(localName)) {
+                        // Entering Build section
+                        inBuild = true;
+
+                    } else if (inBuild) {
+                        // Inside Build section, capture element name
+                        currentElement = localName;
+                    }
+
+                } else if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS && inBuild && currentElement != null) {
+                    // Read text content for Build child elements
+                    String text = reader.getText().trim();
+                    if (!text.isEmpty()) {
+                        switch (currentElement) {
+                            case "BuildID":
+                                this.buildId = text;
+                                break;
+                            case "SourceBasePath":
+                                this.sourceBasePath = text;
+                                break;
+                            case "NumberFiles":
+                                this.numberOfFiles = parseIntegerContent(text);
+                                break;
+                            case "ScanTime":
+                                this.scanTime = parseIntegerContent(text);
+                                break;
+                        }
+                    }
+
+                } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
+                    String localName = reader.getLocalName();
+
+                    if ("Build".equals(localName)) {
+                        // Exiting Build section, we have all needed data
+                        inBuild = false;
+                        // Early exit: we've extracted all needed metadata
+                        if (this.uuid != null) {
+                            break; // Stop parsing, we have everything
+                        }
+
+                    } else if (inBuild) {
+                        // Clear current element when exiting child element
+                        currentElement = null;
+                    }
+                }
+            }
+
+            reader.close();
+
+        } catch (javax.xml.stream.XMLStreamException e) {
+            throw new Exception("Failed to parse audit.fvdl using streaming parser", e);
+        }
+    }
+
+    /**
+     * Helper method to read element text content using StAX reader.
+     * Advances reader to the text content and returns it.
+     *
+     * @param reader XMLStreamReader positioned at START_ELEMENT
+     * @return Text content of the element, or empty string if no text
+     * @throws javax.xml.stream.XMLStreamException if reading fails
+     */
+    private String readElementText(javax.xml.stream.XMLStreamReader reader) throws javax.xml.stream.XMLStreamException {
+        StringBuilder text = new StringBuilder();
+        while (reader.hasNext()) {
+            int event = reader.next();
+            if (event == javax.xml.stream.XMLStreamConstants.CHARACTERS) {
+                text.append(reader.getText());
+            } else if (event == javax.xml.stream.XMLStreamConstants.END_ELEMENT) {
+                break;
+            }
+        }
+        return text.toString().trim();
+    }
+
     private void extractInfoFromAuditFvdl(FprHandle fprHandle) throws Exception {
         Path auditPath = fprHandle.getPath("/audit.fvdl");
 
@@ -127,4 +241,4 @@ public Optional<FilterSet> getDefaultEnabledFilterSet() {
                 .filter(FilterSet::isEnabled)
                 .findFirst();
     }
-}
+}