diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2815b0d847e9..5368a43186a8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -163,4 +163,7 @@ jobs: ENABLED_LANGUAGES: ${{ matrix.name == 'languages' && 'all' || '' }} ROOT: ${{ (matrix.name == 'fixtures' || matrix.name == 'article-api' || matrix.name == 'landings' ) && 'src/fixtures/fixtures' || '' }} TRANSLATIONS_FIXTURE_ROOT: ${{ (matrix.name == 'fixtures' || matrix.name == 'article-api') && 'src/fixtures/fixtures/translations' || '' }} + # Enable debug logging when "Re-run jobs with debug logging" is used in GitHub Actions UI + # This will output additional timing and path information to help diagnose timeout issues + RUNNER_DEBUG: ${{ runner.debug }} run: npm test -- src/${{ matrix.name }}/tests/ diff --git a/content/copilot/index.md b/content/copilot/index.md index 3535e7226fed..13edc7134828 100644 --- a/content/copilot/index.md +++ b/content/copilot/index.md @@ -10,24 +10,6 @@ changelog: introLinks: overview: /copilot/get-started/what-is-github-copilot quickstart: /copilot/get-started/quickstart -featuredLinks: - startHere: - - /copilot/get-started/what-is-github-copilot - - '{% ifversion fpt %}/copilot/get-started/quickstart{% endif %}' - - '{% ifversion fpt %}/copilot/tutorials/try-extensions{% endif %}' - - '{% ifversion fpt %}/copilot/concepts/agents/coding-agent{% endif %}' - - '{% ifversion ghec %}/copilot/get-started/choose-enterprise-plan{% endif %}' - - '{% ifversion ghec %}/copilot/how-tos/set-up/set-up-for-enterprise{% endif %}' - - '{% ifversion ghec %}/copilot/tutorials/coding-agent/pilot-coding-agent{% endif %}' - popular: - - /copilot/get-started/features - - '{% ifversion fpt %}/copilot/tutorials/copilot-chat-cookbook{% endif %}' - - '{% ifversion fpt %}/copilot/how-tos/get-code-suggestions/get-ide-code-suggestions{% endif %}' - - '{% ifversion fpt %}/copilot/how-tos/chat-with-copilot/chat-in-ide{% endif %}' - - '{% ifversion fpt %}/copilot/how-tos/use-copilot-for-common-tasks/use-copilot-in-the-cli{% endif %}' - - '{% ifversion ghec %}/copilot/how-tos/manage-and-track-spending/manage-request-allowances{% endif %}' - - '{% ifversion ghec %}/copilot/tutorials/roll-out-at-scale/enable-developers/drive-adoption{% endif %}' - - '{% ifversion ghec %}/copilot/tutorials/roll-out-at-scale/enable-developers/integrate-ai-agents{% endif %}' layout: discovery-landing heroImage: /assets/images/banner-images/hero-6 versions: diff --git a/content/enterprise-onboarding/index.md b/content/enterprise-onboarding/index.md index c79a08f5d179..535e14c947c5 100644 --- a/content/enterprise-onboarding/index.md +++ b/content/enterprise-onboarding/index.md @@ -1,15 +1,6 @@ --- title: Enterprise onboarding intro: 'Onboard your company to {% data variables.product.prodname_ghe_cloud %} by following our recommended plan. You will set up teams with the access they need, create a policy framework to ensure compliance, and automate processes securely throughout your enterprise.' -featuredLinks: - startHere: - - '/enterprise-onboarding/getting-started-with-your-enterprise' - - '/enterprise-onboarding/adding-users-to-your-enterprise' - - '/enterprise-onboarding/setting-up-organizations-and-teams' - - '/enterprise-onboarding/support-for-your-enterprise' - popular: - - '/enterprise-onboarding/govern-people-and-repositories' - - '/enterprise-onboarding/github-actions-for-your-enterprise' layout: journey-landing journeyTracks: - id: 'getting_started' diff --git a/content/site-policy/privacy-policies/github-codespaces-privacy-statement.md b/content/site-policy/privacy-policies/github-codespaces-privacy-statement.md deleted file mode 100644 index d8839e2e268c..000000000000 --- a/content/site-policy/privacy-policies/github-codespaces-privacy-statement.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: GitHub Codespaces Privacy Statement -redirect_from: - - /github/site-policy/github-codespaces-privacy-policy - - /github/site-policy/github-codespaces-privacy-statement -versions: - fpt: '*' -topics: - - Policy - - Legal ---- - -Effective Date: August 10, 2021 - -Use of GitHub Codespaces is subject to the [GitHub Privacy Statement](/site-policy/privacy-policies/github-privacy-statement). - -Activity on github.dev is subject to [GitHub's Beta Previews terms](/site-policy/github-terms/github-terms-of-service#j-beta-previews) - -## Using {% data variables.product.prodname_vscode %} - -GitHub Codespaces and github.dev allow for use of {% data variables.product.prodname_vscode %} in the web browser. When using {% data variables.product.prodname_vscode_shortname %} in the web browser, some telemetry collection is enabled by default and is [explained in detail on the {% data variables.product.prodname_vscode_shortname %} website](https://code.visualstudio.com/docs/configure/telemetry). Users can opt out of telemetry by going to File > Preferences > Settings under the top left menu. - -If a user chooses to opt out of telemetry capture in {% data variables.product.prodname_vscode_shortname %} while inside of a codespace as outlined, this will sync the disable telemetry preference across all future web sessions in GitHub Codespaces and github.dev. diff --git a/content/site-policy/privacy-policies/github-general-privacy-statement.md b/content/site-policy/privacy-policies/github-general-privacy-statement.md index 5228f566e243..d56ac0f754cf 100644 --- a/content/site-policy/privacy-policies/github-general-privacy-statement.md +++ b/content/site-policy/privacy-policies/github-general-privacy-statement.md @@ -10,6 +10,7 @@ redirect_from: - /github/site-policy/github-privacy-statement - /site-policy/privacy-policies/global-privacy-practices - /site-policy/privacy-policies/github-privacy-statement + - /site-policy/privacy-policies/github-codespaces-privacy-statement versions: fpt: '*' topics: diff --git a/content/site-policy/privacy-policies/index.md b/content/site-policy/privacy-policies/index.md index aef6d92cf7e6..e26a5f1176af 100644 --- a/content/site-policy/privacy-policies/index.md +++ b/content/site-policy/privacy-policies/index.md @@ -9,7 +9,6 @@ children: - /github-general-privacy-statement - /github-subprocessors - /github-cookies - - /github-codespaces-privacy-statement - /github-candidate-privacy-policy redirect_from: - /github/site-policy/github-data-protection-addendum diff --git a/package.json b/package.json index 07747e17f3f6..0217006887a4 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "generate-code-quality-query-list": "tsx src/codeql-queries/scripts/generate-code-quality-query-list.ts", "generate-content-linter-docs": "tsx src/content-linter/scripts/generate-docs.ts", "move-content": "tsx src/content-render/scripts/move-content.ts", + "move-by-content-type": "tsx src/content-render/scripts/move-by-content-type.ts", "openapi-docs": "tsx src/rest/docs.ts", "playwright-test": "playwright test --config src/fixtures/playwright.config.ts --project=\"Google Chrome\"", "lint-report": "tsx src/content-linter/scripts/lint-report.ts", @@ -78,7 +79,7 @@ "release-banner": "tsx src/ghes-releases/scripts/release-banner.ts", "repo-sync": "./src/workflows/local-repo-sync.sh", "reusables": "tsx src/content-render/scripts/reusables-cli.ts", - "resolve-liquid": "tsx src/content-render/scripts/resolve-liquid.ts", + "liquid-tags": "tsx src/content-render/scripts/liquid-tags.ts", "rendered-content-link-checker": "tsx src/links/scripts/rendered-content-link-checker.ts", "rendered-content-link-checker-cli": "tsx src/links/scripts/rendered-content-link-checker-cli.ts", "rest-dev": "tsx src/rest/scripts/update-files.ts", diff --git a/src/ai-tools/lib/file-utils.ts b/src/ai-tools/lib/file-utils.ts index c1cb00d05cb5..af8c0ef91224 100644 --- a/src/ai-tools/lib/file-utils.ts +++ b/src/ai-tools/lib/file-utils.ts @@ -85,7 +85,7 @@ export function mergeFrontmatterProperties(filePath: string, newPropertiesYaml: ) } - if (!parsed.content) { + if (parsed.content === undefined || parsed.content === null) { throw new Error('Failed to parse content from file') } @@ -133,9 +133,11 @@ export function mergeFrontmatterProperties(filePath: string, newPropertiesYaml: const formattedValue = typeof value === 'string' ? `'${value.replace(/'/g, "''")}'` : value // Find the line with this field + let foundField = false for (let i = 1; i < frontmatterEndIndex; i++) { const line = lines[i] if (line.startsWith(`${key}:`)) { + foundField = true // Simple replacement: keep the field name and spacing, replace the value const colonIndex = line.indexOf(':') const leadingSpace = line.substring(colonIndex + 1, colonIndex + 2) // Usually a space @@ -150,6 +152,12 @@ export function mergeFrontmatterProperties(filePath: string, newPropertiesYaml: break } } + + // If field doesn't exist, add it before the closing --- + if (!foundField && frontmatterEndIndex > 0) { + lines.splice(frontmatterEndIndex, 0, `${key}: ${formattedValue}`) + frontmatterEndIndex++ + } } return lines.join('\n') diff --git a/src/ai-tools/lib/prompt-utils.ts b/src/ai-tools/lib/prompt-utils.ts index ef7b726b7b55..7e2fbd887a04 100644 --- a/src/ai-tools/lib/prompt-utils.ts +++ b/src/ai-tools/lib/prompt-utils.ts @@ -2,6 +2,7 @@ import { fileURLToPath } from 'url' import fs from 'fs' import yaml from 'js-yaml' import path from 'path' +import readFrontmatter from '@/frame/lib/read-frontmatter' import { callModelsApi } from '@/ai-tools/lib/call-models-api' export interface PromptMessage { @@ -52,6 +53,67 @@ export function getRefinementDescriptions(editorTypes: string[]): string { return editorTypes.join(', ') } +/** + * Enrich context for intro prompt on index.md files + */ +export function enrichIndexContext(filePath: string, content: string): string { + if (!filePath.endsWith('index.md')) return content + + try { + const { data } = readFrontmatter(content) + if (!data) return content + + // Extract product name from file path (e.g., content/github-models/ -> "GitHub Models") + const productMatch = filePath.replace(/\\/g, '/').match(/content\/([^/]+)/) + const productName = productMatch + ? productMatch[1] + .split('-') + .map((w) => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' ') + : '' + + // Get child article titles + const titles: string[] = [] + if (data.children && Array.isArray(data.children)) { + const dir = path.dirname(filePath) + for (const childPath of data.children.slice(0, 20)) { + try { + const childFile = path.join(dir, `${childPath.replace(/^\//, '')}.md`) + const childContent = fs.readFileSync(childFile, 'utf8') + const { data: childData } = readFrontmatter(childContent) + if (childData?.title) titles.push(childData.title) + } catch (error) { + if (process.env.AI_TOOLS_VERBOSE === 'true') { + console.warn('Failed to read or parse child article for intro context:', { + filePath, + childPath, + error, + }) + } + } + } + } + + // Build context note + const parts: string[] = [] + if (productName) parts.push(`Product: ${productName}`) + if (titles.length > 0) parts.push(`Child articles: ${titles.join(', ')}`) + + if (parts.length > 0) { + return `\n\n---\nContext for intro generation:\n${parts.join('\n')}\n---\n\n${content}` + } + } catch (error) { + if (process.env.AI_TOOLS_VERBOSE === 'true') { + console.warn('Failed to enrich index context for intro generation:', { + filePath, + error, + }) + } + } + + return content +} + /** * Call an editor with the given content and options */ diff --git a/src/ai-tools/prompts/content-type.md b/src/ai-tools/prompts/content-type.md new file mode 100644 index 000000000000..d784cb6a5d66 --- /dev/null +++ b/src/ai-tools/prompts/content-type.md @@ -0,0 +1,38 @@ +Your job is to read through GitHub Docs articles that I provide and figure out what content type it _most_ aligns to and add the frontmatter property `contentType` with an appropriate value. + +**Available `contentType` values (MUST choose from this exact list):** + +- 'get-started' (MANDATORY for files with "quickstart" in the filename; also use for other getting started content) +- 'concepts' (use for files with "about" in the filename; also use for other conceptual content) +- 'how-tos' (use for procedural content AND for subdirectory index.md files that have a `children` array) +- 'rai' (optional - only applies to files with "responsible-use" or "rai" in the filenames) +- 'reference' +- 'tutorials' + +There is one additional type, 'landing', which can ONLY be used on top-level product index.md files: 'content//index.md' + +**CRITICAL RULE**: If a file is an index.md with MORE than three directory parts (e.g., 'content///index.md'), it is a subdirectory index and should use 'how-tos', NOT 'landing'. The fact that it has a `children` array does NOT make it a landing page. + +For prior art, see the following file sets: + +- content/copilot/ +- content/actions/ +- content/account-and-profile/ +- content/integrations/ + +## Output format + +**Important:** Output ONLY the new frontmatter property that should be added to the file. Do not output the entire file content. + +```yaml +contentType: [selected option] +``` + + +**CRITICAL**: You are in write mode. Output ONLY the YAML frontmatter properties to update. +- Return just the YAML property in the format above +- Do NOT include analysis, explanations, or formatting +- Do NOT wrap in markdown code blocks or ```yaml +- Do NOT include the analysis format +- Just return the clean YAML properties for merging + \ No newline at end of file diff --git a/src/ai-tools/prompts/intro.md b/src/ai-tools/prompts/intro.md index 5a0b6be7e972..b6aec3c14da7 100644 --- a/src/ai-tools/prompts/intro.md +++ b/src/ai-tools/prompts/intro.md @@ -1,7 +1,20 @@ -You are an expert SEO content optimizer specializing in GitHub documentation. -Your task is to analyze a GitHub Docs content file and generate or optimize +You are an expert SEO content optimizer specializing in GitHub documentation. +Your task is to analyze a GitHub Docs content file and generate or optimize the intro frontmatter property following Google's meta description best practices. +## Context for index.md files + +For index.md files, you will receive additional context about the product and child articles: +- Product name (e.g., "GitHub Models", "GitHub Copilot") +- List of child article titles + +Use this context to create specific, product-focused intros rather than generic guidance. + +**Examples of good vs generic intros:** +- ❌ "Explore tutorials to build projects and learn new skills with GitHub" +- ❌ "Learn practical guides and step-by-step instructions" +- ✅ "Build AI applications with GitHub Models through hands-on tutorials covering model evaluation and deployment" + ## Core Requirements **Primary constraints (must-haves):** @@ -11,6 +24,11 @@ the intro frontmatter property following Google's meta description best practice * Different approach than title - don't start with same words/phrases * Lists 2-3 concrete outcomes maximum +**For index.md files:** +* Use the provided product name and child article context to create specific intros +* Identify key themes from child article titles to highlight covered topics +* Make intro specific to the product and topics, not generic guidance + **Secondary optimizations (nice-to-haves):** * Include relevant keywords naturally * Version-agnostic ({% data variables.* %} OK, avoid {% ifversion %}) @@ -46,6 +64,15 @@ the intro frontmatter property following Google's meta description best practice ✅ **Uniqueness**: Different angle from article title ✅ **Simplicity**: No colons, no complex lists, flowing sentences +**Examples for index.md files:** + +❌ **Too generic** (ignores provided context): +- Bad: "Explore practical guides and step-by-step instructions to accomplish tasks and solve problems on GitHub" + +✅ **Product-specific** (uses provided context): +- Better: "Learn to use GitHub Models for prototyping, evaluate AI models, and scale deployments across your organization" +- Or: "Build AI-powered applications with GitHub Models, from initial testing to enterprise-scale deployment" + ## Output format Use plain text formatting optimized for terminal readability: @@ -63,23 +90,23 @@ SEO-friendly alternative: "[Single, concise intro that summarizes the article's ## Character limits by content type - **Articles**: Maximum 354 characters -- **Categories**: Maximum 362 characters +- **Categories**: Maximum 362 characters - **Map Topics**: Maximum 362 characters ## Liquid syntax guidelines -**Keep these in intros** (they're acceptable for dynamic content): -- {% data variables.* %} - Product names and variables -- {% data reusables.* %} - Reusable content blocks +**When creating intros from scratch** (no existing intro field): +- Use plain text only - DO NOT use {% data variables.* %} or {% data reusables.* %} syntax +- Write out product names in full (e.g., "GitHub Copilot", "GitHub Actions", "GitHub Docs") +- This prevents hallucinating incorrect variable names -**Avoid these in intros** (version-agnostic content preferred): -- {% ifversion %} blocks - Create intros that work across all supported versions +**When updating existing intros** (intro field already exists): +- Preserve any existing {% data variables.* %} and {% data reusables.* %} references +- You may use the same variable patterns that already appear in the existing intro +- Do not introduce new variable references that weren't in the original -**Common variable meanings** (for analysis purposes): -- {% data variables.product.prodname_github %} = "GitHub" -- {% data variables.product.prodname_ghe_server %} = "GitHub Enterprise Server" -- {% data variables.product.prodname_copilot %} = "GitHub Copilot" -- {% data variables.copilot.copilot_coding_agent %} = "Copilot Coding Agent" +**Always avoid**: +- {% ifversion %} blocks - Create intros that work across all supported versions Focus on creating intros that would make sense to someone discovering this content through Google search, clearly communicating the value and relevance of the article. @@ -89,6 +116,12 @@ Focus on creating intros that would make sense to someone discovering this conte **CRITICAL**: You are in write mode. Output ONLY the YAML frontmatter property to update. +**For index.md files:** +- Use the provided product name and child article context in your intro +- Do NOT write generic intros that could apply to any product +- Make the intro specific to the actual product and covered topics + +**Output format:** - Return just: `intro: "your improved intro text"` - Do NOT include analysis, scoring, explanations, or formatting - Do NOT wrap in markdown code blocks or ```yaml diff --git a/src/ai-tools/scripts/ai-tools.ts b/src/ai-tools/scripts/ai-tools.ts index 8ac78ed62c33..62f2d3a04cfb 100644 --- a/src/ai-tools/scripts/ai-tools.ts +++ b/src/ai-tools/scripts/ai-tools.ts @@ -4,12 +4,14 @@ import path from 'path' import ora from 'ora' import { execFileSync } from 'child_process' import dotenv from 'dotenv' +import readFrontmatter from '@/frame/lib/read-frontmatter' import { findMarkdownFiles, mergeFrontmatterProperties } from '@/ai-tools/lib/file-utils' import { getPromptsDir, getAvailableEditorTypes, getRefinementDescriptions, callEditor, + enrichIndexContext, } from '@/ai-tools/lib/prompt-utils' import { fetchCopilotSpace, convertSpaceToPrompt } from '@/ai-tools/lib/spaces-utils' import { ensureGitHubToken } from '@/ai-tools/lib/auth-utils' @@ -195,13 +197,37 @@ program const relativePath = path.relative(process.cwd(), fileToProcess) spinner.text = `Processing: ${relativePath}` try { - // Resolve Liquid references before processing + // Expand Liquid references before processing + let originalIntro = '' + if (editorType === 'intro') { + const originalContent = fs.readFileSync(fileToProcess, 'utf8') + const { data: originalData } = readFrontmatter(originalContent) + originalIntro = originalData?.intro || '' + } + if (options.verbose) { - console.log(`Resolving Liquid references in: ${relativePath}`) + console.log(`Expanding Liquid references in: ${relativePath}`) + } + runLiquidTagsScript('expand', [fileToProcess], options.verbose || false) + + let content = fs.readFileSync(fileToProcess, 'utf8') + + // For intro prompt, add original intro and enrich context + if (editorType === 'intro') { + if (originalIntro) { + content = `\n\n---\nOriginal intro (unresolved): ${originalIntro}\n---\n\n${content}` + } + content = enrichIndexContext(fileToProcess, content) + } + + // For content-type prompt, skip files that already have contentType + if (editorType === 'content-type' && content.includes('contentType:')) { + spinner.stop() + console.log(`⏭️ Skipping ${relativePath} (already has contentType)`) + runLiquidTagsScript('restore', [fileToProcess], false) + continue } - runResolveLiquid('resolve', [fileToProcess], options.verbose || false) - const content = fs.readFileSync(fileToProcess, 'utf8') const answer = await callEditor( editorType, content, @@ -213,7 +239,7 @@ program spinner.stop() if (options.write) { - if (editorType === 'intro') { + if (editorType === 'intro' || editorType === 'content-type') { // For frontmatter addition/modification, merge properties instead of overwriting entire file const updatedContent = mergeFrontmatterProperties(fileToProcess, answer) fs.writeFileSync(fileToProcess, updatedContent, 'utf8') @@ -235,7 +261,7 @@ program if (options.verbose) { console.log(`Restoring Liquid references in: ${relativePath}`) } - runResolveLiquid('restore', [fileToProcess], options.verbose || false) + runLiquidTagsScript('restore', [fileToProcess], options.verbose || false) } catch (err) { const error = err as Error spinner.fail(`Error processing ${relativePath}: ${error.message}`) @@ -243,7 +269,7 @@ program // Still try to restore Liquid references on error try { - runResolveLiquid('restore', [fileToProcess], false) + runLiquidTagsScript('restore', [fileToProcess], false) } catch (restoreError) { // Log restore failures in verbose mode for debugging if (options.verbose) { @@ -275,35 +301,32 @@ program program.parse(process.argv) /** - * Run resolve-liquid command on specified file paths + * Run liquid-tags command on specified file paths */ -function runResolveLiquid( - command: 'resolve' | 'restore', +function runLiquidTagsScript( + command: 'expand' | 'restore', filePaths: string[], verbose: boolean = false, ): void { const args = [command, '--paths', ...filePaths] - if (command === 'resolve') { - args.push('--recursive') - } if (verbose) { args.push('--verbose') } try { - // Run resolve-liquid via tsx - const resolveLiquidPath = path.join( + // Run liquid-tags script via tsx + const liquidTagsScriptPath = path.join( process.cwd(), - 'src/content-render/scripts/resolve-liquid.ts', + 'src/content-render/scripts/liquid-tags.ts', ) - execFileSync('npx', ['tsx', resolveLiquidPath, ...args], { + execFileSync('npx', ['tsx', liquidTagsScriptPath, ...args], { stdio: verbose ? 'inherit' : 'pipe', }) } catch (error) { if (verbose) { - console.error(`Error running resolve-liquid ${command}:`, error) + console.error(`Error running liquid-tags ${command}:`, error) } - // Don't fail the entire process if resolve-liquid fails + // Don't fail the entire process if liquid-tags fails } } diff --git a/src/article-api/lib/get-link-data.ts b/src/article-api/lib/get-link-data.ts new file mode 100644 index 000000000000..f9c33c74608e --- /dev/null +++ b/src/article-api/lib/get-link-data.ts @@ -0,0 +1,48 @@ +import type { Context, Page } from '@/types' +import type { LinkData } from '@/article-api/transformers/types' + +/** + * Resolves link data (title, href, intro) for a given href and page + * + * This helper is used by landing page transformers to build link lists. + * It resolves the page from an href, renders its title and intro, and + * returns the canonical permalink. + * + * @param href - The href to resolve (can be relative or absolute) + * @param languageCode - The language code for the current page + * @param pathname - The current page's pathname (for relative resolution) + * @param context - The rendering context + * @param resolvePath - Function to resolve an href to a Page object + * @returns LinkData with resolved title, href, and optional intro + */ +export async function getLinkData( + href: string, + languageCode: string, + pathname: string, + context: Context, + resolvePath: ( + href: string, + languageCode: string, + pathname: string, + context: Context, + ) => Page | undefined, +): Promise { + const linkedPage = resolvePath(href, languageCode, pathname, context) + if (!linkedPage) return { href, title: href } + + const title = await linkedPage.renderTitle(context, { unwrap: true }) + const intro = linkedPage.intro + ? await linkedPage.renderProp('intro', context, { textOnly: true }) + : '' + + const permalink = linkedPage.permalinks.find( + (p) => p.languageCode === languageCode && p.pageVersion === context.currentVersion, + ) + const resolvedHref = permalink ? permalink.href : href + + return { + href: resolvedHref, + title, + intro, + } +} diff --git a/src/article-api/lib/load-template.ts b/src/article-api/lib/load-template.ts new file mode 100644 index 000000000000..2cf169ba791b --- /dev/null +++ b/src/article-api/lib/load-template.ts @@ -0,0 +1,31 @@ +import { readFileSync } from 'fs' +import { join, dirname } from 'path' +import { fileURLToPath } from 'url' + +// Get the directory path for the transformers directory +// This will be used to resolve template paths relative to transformers +const __filename = fileURLToPath(import.meta.url) +const __dirname = dirname(__filename) + +/** + * Load a template file from the templates directory + * + * This helper loads Liquid template files used by transformers. + * Templates are located in src/article-api/templates/ + * + * @param templateName - The name of the template file (e.g., 'landing-page.template.md') + * @returns The template content as a string + * + * @example + * ```typescript + * const template = loadTemplate('landing-page.template.md') + * const rendered = await renderContent(template, context) + * ``` + */ +export function loadTemplate(templateName: string): string { + // Templates are in ../templates relative to the lib directory + // lib is at src/article-api/lib + // templates is at src/article-api/templates + const templatePath = join(__dirname, '../templates', templateName) + return readFileSync(templatePath, 'utf8') +} diff --git a/src/article-api/lib/resolve-path.ts b/src/article-api/lib/resolve-path.ts new file mode 100644 index 000000000000..88acc8f618b1 --- /dev/null +++ b/src/article-api/lib/resolve-path.ts @@ -0,0 +1,107 @@ +import type { Context, Page } from '@/types' + +/** + * Resolves an href to a Page object from the context + * + * This function handles various href formats: + * - External URLs (http/https) - returns undefined + * - Language-prefixed absolute paths (/en/copilot/...) - direct lookup + * - Absolute paths without language (/copilot/...) - adds language prefix + * - Relative paths (get-started) - resolved relative to pathname + * + * The function searches through context.pages using multiple strategies: + * 1. Direct key lookup with language prefix + * 2. Relative path joining with current pathname + * 3. endsWith matching for versioned keys (e.g., /en/enterprise-cloud@latest/...) + * + * @param href - The href to resolve + * @param languageCode - The language code (e.g., 'en') + * @param pathname - The current page's pathname (e.g., '/en/copilot') + * @param context - The rendering context containing all pages + * @returns The resolved Page object, or undefined if not found + * + * @example + * ```typescript + * // Absolute path with language + * resolvePath('/en/copilot/quickstart', 'en', '/en/copilot', context) + * + * // Absolute path without language (adds /en/) + * resolvePath('/copilot/quickstart', 'en', '/en/copilot', context) + * + * // Relative path (resolves to /en/copilot/quickstart) + * resolvePath('quickstart', 'en', '/en/copilot', context) + * + * // Relative path with leading slash (resolves relative to pathname) + * resolvePath('/quickstart', 'en', '/en/copilot', context) // -> /en/copilot/quickstart + * ``` + */ +export function resolvePath( + href: string, + languageCode: string, + pathname: string, + context: Context, +): Page | undefined { + // External URLs cannot be resolved + if (href.startsWith('http://') || href.startsWith('https://')) { + return undefined + } + + if (!context.pages) { + return undefined + } + + // Normalize href to start with / + const normalizedHref = href.startsWith('/') ? href : `/${href}` + + // Build full path with language prefix if needed + let fullPath: string + if (normalizedHref.startsWith(`/${languageCode}/`)) { + // Already has language prefix + fullPath = normalizedHref + } else if (href.startsWith('/') && !href.startsWith(`/${languageCode}/`)) { + // Path with leading slash but no language prefix - treat as relative to pathname + // e.g., pathname='/en/copilot', href='/get-started' -> '/en/copilot/get-started' + fullPath = pathname + href + } else { + // Relative path - add language prefix + // e.g., href='quickstart' -> '/en/quickstart' + fullPath = `/${languageCode}${normalizedHref}` + } + + // Clean up trailing slashes + const cleanPath = fullPath.replace(/\/$/, '') + + // Strategy 1: Direct lookup + if (context.pages[cleanPath]) { + return context.pages[cleanPath] + } + + // Strategy 2: Try relative to current pathname + const currentPath = pathname.replace(/\/$/, '') + const relativeHref = href.startsWith('/') ? href.slice(1) : href + const joinedPath = `${currentPath}/${relativeHref}` + + if (context.pages[joinedPath]) { + return context.pages[joinedPath] + } + + // Strategy 3: Search for keys that end with the path (handles versioned keys) + // e.g., key='/en/enterprise-cloud@latest/copilot' should match path='/en/copilot' + for (const [key, page] of Object.entries(context.pages)) { + if (key.endsWith(cleanPath) || key.endsWith(`${cleanPath}/`)) { + return page + } + } + + // Strategy 4: If href started with /, try endsWith matching on that too + if (href.startsWith('/')) { + const hrefClean = href.replace(/\/$/, '') + for (const [key, page] of Object.entries(context.pages)) { + if (key.endsWith(hrefClean) || key.endsWith(`${hrefClean}/`)) { + return page + } + } + } + + return undefined +} diff --git a/src/article-api/templates/landing-page.template.md b/src/article-api/templates/landing-page.template.md new file mode 100644 index 000000000000..cf0dac2168d1 --- /dev/null +++ b/src/article-api/templates/landing-page.template.md @@ -0,0 +1,25 @@ +# {{ title }} + +{% if intro %} +{{ intro }} +{% endif %} + +{% for section in sections %} +{% if section.title %} +## {{ section.title }} +{% endif %} + +{% for group in section.groups %} +{% if group.title %} +### {{ group.title }} +{% endif %} + +{% for link in group.links %} +* [{{ link.title }}]({{ link.href }}) +{% if link.intro %} + {{ link.intro }} +{% endif %} +{% endfor %} + +{% endfor %} +{% endfor %} \ No newline at end of file diff --git a/src/article-api/tests/get-link-data.ts b/src/article-api/tests/get-link-data.ts new file mode 100644 index 000000000000..fc11254faba3 --- /dev/null +++ b/src/article-api/tests/get-link-data.ts @@ -0,0 +1,136 @@ +import { describe, expect, test, vi } from 'vitest' +import { getLinkData } from '@/article-api/lib/get-link-data' +import type { Context, Page, Permalink } from '@/types' + +// Helper to create a minimal mock page +function createMockPage(options: { + title?: string + intro?: string + permalinks?: Partial[] +}): Page { + const page = { + title: options.title || 'Test Title', + intro: options.intro, + permalinks: (options.permalinks || []) as Permalink[], + renderTitle: vi.fn().mockResolvedValue(options.title || 'Test Title'), + renderProp: vi.fn().mockResolvedValue(options.intro || ''), + } + return page as unknown as Page +} + +// Helper to create a minimal context +function createContext(currentVersion = 'free-pro-team@latest'): Context { + return { currentVersion } as unknown as Context +} + +describe('getLinkData', () => { + describe('when page is not found', () => { + test('returns href as both href and title when page not resolved', async () => { + const resolvePath = vi.fn().mockReturnValue(undefined) + const context = createContext() + + const result = await getLinkData( + '/en/missing-page', + 'en', + '/en/current', + context, + resolvePath, + ) + + expect(result).toEqual({ + href: '/en/missing-page', + title: '/en/missing-page', + }) + }) + }) + + describe('when page is found', () => { + test('returns rendered title from page', async () => { + const page = createMockPage({ title: 'My Page Title' }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext() + + const result = await getLinkData('/en/some-page', 'en', '/en/current', context, resolvePath) + + expect(result.title).toBe('My Page Title') + expect(page.renderTitle).toHaveBeenCalledWith(context, { unwrap: true }) + }) + + test('returns rendered intro when page has intro', async () => { + const page = createMockPage({ + title: 'Page', + intro: 'This is the intro text', + }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext() + + const result = await getLinkData('/en/some-page', 'en', '/en/current', context, resolvePath) + + expect(result.intro).toBe('This is the intro text') + expect(page.renderProp).toHaveBeenCalledWith('intro', context, { textOnly: true }) + }) + + test('returns empty intro when page has no intro', async () => { + const page = createMockPage({ title: 'Page', intro: undefined }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext() + + const result = await getLinkData('/en/some-page', 'en', '/en/current', context, resolvePath) + + expect(result.intro).toBe('') + expect(page.renderProp).not.toHaveBeenCalled() + }) + + test('uses permalink href when matching permalink found', async () => { + const page = createMockPage({ + title: 'Page', + permalinks: [ + { languageCode: 'en', pageVersion: 'free-pro-team@latest', href: '/en/resolved-path' }, + ], + }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext('free-pro-team@latest') + + const result = await getLinkData( + '/en/original-href', + 'en', + '/en/current', + context, + resolvePath, + ) + + expect(result.href).toBe('/en/resolved-path') + }) + + test('falls back to original href when no matching permalink', async () => { + const page = createMockPage({ + title: 'Page', + permalinks: [{ languageCode: 'ja', pageVersion: 'free-pro-team@latest', href: '/ja/page' }], + }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext('free-pro-team@latest') + + const result = await getLinkData( + '/en/original-href', + 'en', + '/en/current', + context, + resolvePath, + ) + + expect(result.href).toBe('/en/original-href') + }) + }) + + describe('resolvePath function usage', () => { + test('passes correct arguments to resolvePath', async () => { + const page = createMockPage({ title: 'Page' }) + const resolvePath = vi.fn().mockReturnValue(page) + const context = createContext() + + await getLinkData('/en/target', 'en', '/en/current', context, resolvePath) + + expect(resolvePath).toHaveBeenCalledWith('/en/target', 'en', '/en/current', context) + }) + }) +}) diff --git a/src/article-api/tests/github-apps-transformer.ts b/src/article-api/tests/github-apps-transformer.ts index 003ea686345a..9a926d3d44c2 100644 --- a/src/article-api/tests/github-apps-transformer.ts +++ b/src/article-api/tests/github-apps-transformer.ts @@ -70,11 +70,15 @@ describe('GitHub Apps transformer', () => { }) test('endpoints are formatted as bullet lists', async () => { - const res = await get( - makeURL( - '/en/rest/authentication/endpoints-available-for-github-app-installation-access-tokens', - ), + const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1' + const url = makeURL( + '/en/rest/authentication/endpoints-available-for-github-app-installation-access-tokens', ) + const startTime = DEBUG ? Date.now() : 0 + if (DEBUG) console.log(`[DEBUG] Test sending request to ${url}`) + const res = await get(url) + if (DEBUG) + console.log(`[DEBUG] Test response: ${res.statusCode} in ${Date.now() - startTime}ms`) expect(res.statusCode).toBe(200) // Check for bullet list items with asterisks (per content guidelines) diff --git a/src/article-api/tests/load-template.ts b/src/article-api/tests/load-template.ts new file mode 100644 index 000000000000..ade3d72fadf6 --- /dev/null +++ b/src/article-api/tests/load-template.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from 'vitest' +import { loadTemplate } from '@/article-api/lib/load-template' + +describe('loadTemplate', () => { + test('loads an existing template file', () => { + const content = loadTemplate('landing-page.template.md') + expect(content).toContain('# {{ title }}') + expect(content).toContain('{% for section in sections %}') + }) + + test('throws error for non-existent template', () => { + expect(() => loadTemplate('non-existent-template.md')).toThrow() + }) + + test('returns template content as string', () => { + const content = loadTemplate('landing-page.template.md') + expect(typeof content).toBe('string') + expect(content.length).toBeGreaterThan(0) + }) +}) diff --git a/src/article-api/tests/resolve-path.ts b/src/article-api/tests/resolve-path.ts new file mode 100644 index 000000000000..6156b8c1e689 --- /dev/null +++ b/src/article-api/tests/resolve-path.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from 'vitest' +import { resolvePath } from '@/article-api/lib/resolve-path' +import type { Context, Page } from '@/types' + +// Helper to create a minimal mock page +function createMockPage(relativePath: string): Page { + return { + relativePath, + permalinks: [], + } as unknown as Page +} + +// Helper to create a minimal context with pages +function createContext(pages: Record): Context { + return { pages } as unknown as Context +} + +describe('resolvePath', () => { + describe('external URLs', () => { + test('returns undefined for http URLs', () => { + const context = createContext({}) + const result = resolvePath('http://example.com', 'en', '/en/copilot', context) + expect(result).toBeUndefined() + }) + + test('returns undefined for https URLs', () => { + const context = createContext({}) + const result = resolvePath('https://example.com/path', 'en', '/en/copilot', context) + expect(result).toBeUndefined() + }) + }) + + describe('missing context.pages', () => { + test('returns undefined when context.pages is undefined', () => { + const context = {} as Context + const result = resolvePath('/en/copilot', 'en', '/en', context) + expect(result).toBeUndefined() + }) + }) + + describe('language-prefixed absolute paths', () => { + test('finds page with exact language-prefixed path', () => { + const page = createMockPage('copilot/quickstart.md') + const context = createContext({ + '/en/copilot/quickstart': page, + }) + const result = resolvePath('/en/copilot/quickstart', 'en', '/en/copilot', context) + expect(result).toBe(page) + }) + + test('handles trailing slash in href', () => { + const page = createMockPage('copilot/quickstart.md') + const context = createContext({ + '/en/copilot/quickstart': page, + }) + const result = resolvePath('/en/copilot/quickstart/', 'en', '/en/copilot', context) + expect(result).toBe(page) + }) + }) + + describe('paths with leading slash (relative to pathname)', () => { + test('resolves path relative to current pathname', () => { + const page = createMockPage('copilot/get-started.md') + const context = createContext({ + '/en/copilot/get-started': page, + }) + const result = resolvePath('/get-started', 'en', '/en/copilot', context) + expect(result).toBe(page) + }) + + test('handles nested paths relative to pathname', () => { + const page = createMockPage('copilot/tutorials/basics.md') + const context = createContext({ + '/en/copilot/tutorials/basics': page, + }) + const result = resolvePath('/tutorials/basics', 'en', '/en/copilot', context) + expect(result).toBe(page) + }) + }) + + describe('relative paths without leading slash', () => { + test('resolves relative path by joining with pathname', () => { + const page = createMockPage('copilot/quickstart.md') + const context = createContext({ + '/en/copilot/quickstart': page, + }) + const result = resolvePath('quickstart', 'en', '/en/copilot', context) + expect(result).toBe(page) + }) + }) + + describe('versioned keys (endsWith matching)', () => { + test('finds versioned page using endsWith strategy', () => { + const page = createMockPage('copilot/index.md') + // The key ends with the path portion after the version string + const context = createContext({ + '/en/enterprise-cloud@latest/copilot': page, + }) + // When looking for /copilot, strategy 4 will find keys ending with /copilot + const result = resolvePath('/copilot', 'en', '/en', context) + expect(result).toBe(page) + }) + + test('finds page when key has trailing slash', () => { + const page = createMockPage('copilot/index.md') + const context = createContext({ + '/en/copilot/': page, + }) + const result = resolvePath('/en/copilot', 'en', '/en', context) + expect(result).toBe(page) + }) + }) + + describe('not found cases', () => { + test('returns undefined when page does not exist', () => { + const context = createContext({ + '/en/other': createMockPage('other.md'), + }) + const result = resolvePath('/en/copilot', 'en', '/en', context) + expect(result).toBeUndefined() + }) + }) +}) diff --git a/src/article-api/tests/rest-transformer.ts b/src/article-api/tests/rest-transformer.ts index f306102197f6..3cd8e6d5ad85 100644 --- a/src/article-api/tests/rest-transformer.ts +++ b/src/article-api/tests/rest-transformer.ts @@ -81,7 +81,13 @@ describe('REST transformer', () => { }) test('Status codes are formatted correctly', async () => { - const res = await get(makeURL('/en/rest/actions/artifacts')) + const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1' + const url = makeURL('/en/rest/actions/artifacts') + const startTime = DEBUG ? Date.now() : 0 + if (DEBUG) console.log(`[DEBUG] Test sending request to ${url}`) + const res = await get(url) + if (DEBUG) + console.log(`[DEBUG] Test response: ${res.statusCode} in ${Date.now() - startTime}ms`) expect(res.statusCode).toBe(200) // Check for status codes section diff --git a/src/article-api/tests/toc-transformer.ts b/src/article-api/tests/toc-transformer.ts new file mode 100644 index 000000000000..4bef199a0879 --- /dev/null +++ b/src/article-api/tests/toc-transformer.ts @@ -0,0 +1,55 @@ +import { describe, expect, test } from 'vitest' + +import { get } from '@/tests/helpers/e2etest' + +const makeURL = (pathname: string): string => + `/api/article/body?${new URLSearchParams({ pathname })}` + +describe('toc transformer', () => { + test('renders a category page (3-segment URL) with children', async () => { + // /en/actions/category is a category page (documentType: category) with children but no layout + const res = await get(makeURL('/en/actions/category')) + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toContain('text/markdown') + + // Check for title and intro + expect(res.body).toContain('# Category page of GitHub Actions') + expect(res.body).toContain('Learn how to migrate your existing CI/CD workflows') + + // Should have Links section with children (uses full title, not shortTitle) + expect(res.body).toContain('## Links') + expect(res.body).toContain('[Subcategory page about Actions](/en/actions/category/subcategory)') + }) + + test('renders a subcategory page (4+-segment URL) with children', async () => { + // /en/actions/category/subcategory is a subcategory page (documentType: subcategory) with children but no layout + const res = await get(makeURL('/en/actions/category/subcategory')) + expect(res.statusCode).toBe(200) + expect(res.headers['content-type']).toContain('text/markdown') + + // Check for title + expect(res.body).toContain('# Subcategory page about Actions') + + // Should have Links section with children + expect(res.body).toContain('## Links') + }) + + test('includes child intro text', async () => { + const res = await get(makeURL('/en/actions/category')) + expect(res.statusCode).toBe(200) + + // Each child link should have its intro text below + expect(res.body).toContain("Here's the intro for") + }) + + test('resolves child page titles correctly', async () => { + const res = await get(makeURL('/en/actions/category/subcategory')) + expect(res.statusCode).toBe(200) + + // Should NOT have href paths as titles + expect(res.body).not.toContain('* [/en/actions/category/subcategory/') + + // Should have proper article titles (or shortTitle) + expect(res.body).toMatch(/\[.*\]\(\/en\/actions\/category\/subcategory\/.*\)/) + }) +}) diff --git a/src/article-api/transformers/github-apps-transformer.ts b/src/article-api/transformers/github-apps-transformer.ts index 0d6a4732a1ab..522c9873b50d 100644 --- a/src/article-api/transformers/github-apps-transformer.ts +++ b/src/article-api/transformers/github-apps-transformer.ts @@ -8,6 +8,7 @@ import { fileURLToPath } from 'url' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) +const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1' // GitHub Apps data types interface GitHubAppsOperation { @@ -100,6 +101,9 @@ export class GithubAppsTransformer implements PageTransformer { context: Context, apiVersion?: string, ): Promise { + const startTime = DEBUG ? Date.now() : 0 + if (DEBUG) console.log(`[DEBUG] GitHubAppsTransformer: ${pathname}`) + // Import getAppsData dynamically to avoid circular dependencies const { getAppsData } = await import('@/github-apps/lib/index') @@ -216,6 +220,10 @@ export class GithubAppsTransformer implements PageTransformer { }) } + if (DEBUG) + console.log( + `[DEBUG] GitHubAppsTransformer.transform completed in ${Date.now() - startTime}ms`, + ) return finalContent } diff --git a/src/article-api/transformers/index.ts b/src/article-api/transformers/index.ts index 589439ed8ea7..7974441e18fa 100644 --- a/src/article-api/transformers/index.ts +++ b/src/article-api/transformers/index.ts @@ -6,6 +6,7 @@ import { AuditLogsTransformer } from './audit-logs-transformer' import { GraphQLTransformer } from './graphql-transformer' import { GithubAppsTransformer } from './github-apps-transformer' import { WebhooksTransformer } from './webhooks-transformer' +import { TocTransformer } from './toc-transformer' /** * Global transformer registry @@ -20,6 +21,7 @@ transformerRegistry.register(new AuditLogsTransformer()) transformerRegistry.register(new GraphQLTransformer()) transformerRegistry.register(new GithubAppsTransformer()) transformerRegistry.register(new WebhooksTransformer()) +transformerRegistry.register(new TocTransformer()) export { TransformerRegistry } from './types' export type { PageTransformer } from './types' diff --git a/src/article-api/transformers/rest-transformer.ts b/src/article-api/transformers/rest-transformer.ts index 87f6b9abd417..5ba981f1a55d 100644 --- a/src/article-api/transformers/rest-transformer.ts +++ b/src/article-api/transformers/rest-transformer.ts @@ -10,6 +10,7 @@ import { fastTextOnly } from '@/content-render/unified/text-only' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) +const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1' /** * Transformer for REST API pages @@ -28,6 +29,9 @@ export class RestTransformer implements PageTransformer { context: Context, apiVersion?: string, ): Promise { + const startTime = DEBUG ? Date.now() : 0 + if (DEBUG) console.log(`[DEBUG] RestTransformer: ${pathname}`) + // Import getRest dynamically to avoid circular dependencies const { default: getRest } = await import('@/rest/lib/index') @@ -110,6 +114,7 @@ export class RestTransformer implements PageTransformer { markdownRequested: true, }) + if (DEBUG) console.log(`[DEBUG] RestTransformer completed in ${Date.now() - startTime}ms`) return rendered } diff --git a/src/article-api/transformers/toc-transformer.ts b/src/article-api/transformers/toc-transformer.ts new file mode 100644 index 000000000000..f74c38a7fa31 --- /dev/null +++ b/src/article-api/transformers/toc-transformer.ts @@ -0,0 +1,78 @@ +import type { Context, Page } from '@/types' +import type { PageTransformer, TemplateData, Section } from './types' +import { renderContent } from '@/content-render/index' +import { loadTemplate } from '@/article-api/lib/load-template' +import { resolvePath } from '@/article-api/lib/resolve-path' +import { getLinkData } from '@/article-api/lib/get-link-data' + +interface CategoryPage extends Page { + children?: string[] +} + +/** + * Transformer for table of contents (TOC) landing pages - pages with children but no specific layout. + * These are simple navigation pages (category/subcategory) that list child pages with titles and intros. + * Corresponds to TocLanding component in the web UI. + */ +export class TocTransformer implements PageTransformer { + templateName = 'landing-page.template.md' + + canTransform(page: Page): boolean { + // Transform pages that have children but no layout specified + // These are typically category or subcategory pages + const categoryPage = page as CategoryPage + return ( + !page.layout && + (page.documentType === 'category' || page.documentType === 'subcategory') && + !!categoryPage.children && + categoryPage.children.length > 0 + ) + } + + async transform(page: Page, pathname: string, context: Context): Promise { + const templateData = await this.prepareTemplateData(page, pathname, context) + const templateContent = loadTemplate(this.templateName) + + return await renderContent(templateContent, { + ...context, + ...templateData, + markdownRequested: true, + }) + } + + private async prepareTemplateData( + page: Page, + pathname: string, + context: Context, + ): Promise { + const mapPage = page as CategoryPage + const languageCode = page.languageCode || 'en' + const sections: Section[] = [] + + // Get all child pages + if (mapPage.children?.length) { + const links = await Promise.all( + mapPage.children.map(async (childHref) => { + return await getLinkData(childHref, languageCode, pathname, context, resolvePath) + }), + ) + + const validLinks = links.filter((l) => l.href) + if (validLinks.length > 0) { + sections.push({ + title: 'Links', + groups: [{ title: null, links: validLinks }], + }) + } + } + + const intro = page.intro ? await page.renderProp('intro', context, { textOnly: true }) : '' + const title = await page.renderTitle(context, { unwrap: true }) + + return { + title, + intro, + sections, + } + } +} diff --git a/src/article-api/transformers/types.ts b/src/article-api/transformers/types.ts index 4916aea2283e..aa2441f816e2 100644 --- a/src/article-api/transformers/types.ts +++ b/src/article-api/transformers/types.ts @@ -1,5 +1,39 @@ import type { Context, Page } from '@/types' +/** + * Link data for landing page sections + */ +export interface LinkData { + href: string + title: string + intro?: string +} + +/** + * Group of links with an optional title + */ +export interface LinkGroup { + title: string | null + links: LinkData[] +} + +/** + * Section containing multiple link groups + */ +export interface Section { + title: string | null + groups: LinkGroup[] +} + +/** + * Template data structure for landing pages + */ +export interface TemplateData { + title: string + intro: string + sections: Section[] +} + /** * Base interface for page-to-markdown transformers * @@ -7,6 +41,13 @@ import type { Context, Page } from '@/types' * into markdown format for the Article API */ export interface PageTransformer { + /** + * Optional: The name of the template file to use for rendering + * If provided, can be used by helper functions to load the template + * Example: 'landing-page.template.md' + */ + templateName?: string + /** * Check if this transformer can handle the given page */ diff --git a/src/content-render/scripts/resolve-liquid.ts b/src/content-render/scripts/liquid-tags.ts similarity index 78% rename from src/content-render/scripts/resolve-liquid.ts rename to src/content-render/scripts/liquid-tags.ts index b76905842b4f..7854b05b7d68 100644 --- a/src/content-render/scripts/resolve-liquid.ts +++ b/src/content-render/scripts/liquid-tags.ts @@ -1,9 +1,9 @@ /* * @purpose Writer tool - * @description Resolve and unresolve Liquid data references in content files + * @description Expand and restore Liquid data references in content files */ -// Usage: npm run resolve-liquid -- resolve --paths content/pull-requests/about.md -// Usage: npm run resolve-liquid -- restore --paths content/pull-requests/about.md +// Usage: npm run liquid-tags -- expand --paths content/pull-requests/about.md +// Usage: npm run liquid-tags -- restore --paths content/pull-requests/about.md import { Command } from 'commander' import fs from 'fs' @@ -12,14 +12,14 @@ import yaml from 'js-yaml' import chalk from 'chalk' // Type definitions -interface ResolveOptions { +interface ExpandOptions { paths: string[] verbose?: boolean markers?: boolean dryRun?: boolean reusablesOnly?: boolean variablesOnly?: boolean - recursive?: boolean + shallow?: boolean } interface LiquidReference { @@ -36,38 +36,88 @@ const DATA_ROOT = path.resolve(path.join(ROOT, 'data')) const REUSABLES_ROOT = path.join(DATA_ROOT, 'reusables') const VARIABLES_ROOT = path.join(DATA_ROOT, 'variables') -// Regex pattern to match resolved content blocks -const RESOLVED_PATTERN = - /(.+?)/gs +// Regex pattern to match expanded content blocks +const EXPANDED_PATTERN = /(.+?)/gs /** * Get the file path for a data reference + * + * Validates and normalizes the incoming dataPath to prevent path traversal + * and ensure the final resolved path remains within the expected root. */ function getDataFilePath(type: 'reusable' | 'variable', dataPath: string): string { + // Basic validation of the raw dataPath + if (path.isAbsolute(dataPath)) { + throw new Error(`Invalid ${type} data path: absolute paths are not allowed: ${dataPath}`) + } + + // Disallow path traversal and empty segments + const segments = dataPath.split(/[\\/]/) + if (segments.some((segment) => segment === '..' || segment === '')) { + throw new Error(`Invalid ${type} data path: contains disallowed segments: ${dataPath}`) + } + + // Restrict allowed characters to a conservative safe set + if (!/^[A-Za-z0-9_.\-/]+$/.test(dataPath)) { + throw new Error(`Invalid ${type} data path: contains disallowed characters: ${dataPath}`) + } + if (type === 'reusable') { - return path.join(REUSABLES_ROOT, `${dataPath.replace(/\./g, '/')}.md`) + const baseRoot = path.resolve(REUSABLES_ROOT) + const relativePath = dataPath.replace(/\./g, '/') + const candidatePath = path.resolve(baseRoot, `${relativePath}.md`) + + if (!candidatePath.startsWith(baseRoot + path.sep)) { + throw new Error(`Invalid reusable data path: escapes reusables root: ${dataPath}`) + } + + return candidatePath } else { + const baseRoot = path.resolve(VARIABLES_ROOT) const fileName = dataPath.split('.')[0] - return path.join(VARIABLES_ROOT, `${fileName}.yml`) + const candidatePath = path.resolve(baseRoot, `${fileName}.yml`) + + if (!candidatePath.startsWith(baseRoot + path.sep)) { + throw new Error(`Invalid variable data path: escapes variables root: ${dataPath}`) + } + + return candidatePath + } +} + +/** + * Convert a file path back to data path format (for consistent verbose output) + */ +function convertFilePathToDataPath(filePath: string): string { + const normalizedPath = path.normalize(filePath) + + if (normalizedPath.includes('reusables')) { + const relativePath = path.relative(REUSABLES_ROOT, normalizedPath) + return `reusables.${relativePath.replace(/\.md$/, '').replace(/[\\/]/g, '.')}` + } else if (normalizedPath.includes('variables')) { + const relativePath = path.relative(VARIABLES_ROOT, normalizedPath) + return `variables.${relativePath.replace(/\.yml$/, '').replace(/[\\/]/g, '.')}` } + + return filePath } const program = new Command() program - .name('resolve-liquid') - .description('Tools to resolve and unresolve Liquid data references in content files') + .name('liquid-tags') + .description('Tools to expand and restore Liquid data references in content files') program - .command('resolve') - .description('Resolve {% data reusables %} and {% data variables %} statements to their content') + .command('expand') + .description('Expand {% data reusables %} and {% data variables %} statements to their content') .option('--paths ', 'Content file paths to process', []) .option('-v, --verbose', 'Verbose output', false) .option('--no-markers', 'Skip HTML comment markers (output cannot be restored to Liquid)', true) .option('--reusables-only', 'Process only reusables (skip variables)', false) .option('--variables-only', 'Process only variables (skip reusables)', false) - .option('-r, --recursive', 'Keep resolving until no references remain (max 10 iterations)', false) - .action((options: ResolveOptions) => resolveReferences(options)) + .option('--shallow', 'Expand only one level (do not expand nested references)', false) + .action((options: ExpandOptions) => expandReferences(options)) program .command('restore') @@ -76,14 +126,14 @@ program .option('-v, --verbose', 'Verbose output', false) .option('--reusables-only', 'Process only reusables (skip variables)', false) .option('--variables-only', 'Process only variables (skip reusables)', false) - .action((options: ResolveOptions) => restoreReferences(options)) + .action((options: ExpandOptions) => restoreReferences(options)) program.parse() /** * Get allowed types based on command options */ -function getAllowedTypes(options: ResolveOptions): Array<'reusable' | 'variable'> { +function getAllowedTypes(options: ExpandOptions): Array<'reusable' | 'variable'> { if (options.reusablesOnly && options.variablesOnly) { console.log( chalk.yellow( @@ -106,14 +156,15 @@ function getAllowedTypes(options: ResolveOptions): Array<'reusable' | 'variable' } /** - * Resolve Liquid data references in content files + * Expand Liquid data references in content files */ -async function resolveReferences(options: ResolveOptions): Promise { - const { paths, verbose, markers, recursive } = options +async function expandReferences(options: ExpandOptions): Promise { + const { paths, verbose, markers, shallow } = options // markers will be true by default, false when --no-markers is used const withMarkers = markers !== false + const recursive = !shallow // Recursive by default unless --shallow is specified const allowedTypes = getAllowedTypes(options) - const maxIterations = 10 // Safety limit for recursive resolution + const maxIterations = 10 // Safety limit for recursive expansion if (paths.length === 0) { console.error(chalk.red('Error: No paths provided. Use --paths option.')) @@ -143,7 +194,7 @@ async function resolveReferences(options: ResolveOptions): Promise { } const content = fs.readFileSync(filePath, 'utf-8') - const resolvedContent = await resolveFileContent( + const expandedContent = await expandFileContent( content, filePath, verbose, @@ -151,10 +202,10 @@ async function resolveReferences(options: ResolveOptions): Promise { allowedTypes, ) - if (resolvedContent !== content) { - fs.writeFileSync(filePath, resolvedContent, 'utf-8') + if (expandedContent !== content) { + fs.writeFileSync(filePath, expandedContent, 'utf-8') if (iteration === 1 || !recursive) { - console.log(chalk.green(`✓ Resolved references in: ${filePath}`)) + console.log(chalk.green(`✓ Expanded references in: ${filePath}`)) } } else { if (verbose && iteration === 1) { @@ -163,11 +214,11 @@ async function resolveReferences(options: ResolveOptions): Promise { } // Check for remaining references - const remainingRefs = findLiquidReferences(resolvedContent, allowedTypes) + const remainingRefs = findLiquidReferences(expandedContent, allowedTypes) hasRemainingRefs = remainingRefs.length > 0 - if (!recursive) { - // Non-recursive mode: show remaining references and break + if (shallow) { + // Shallow mode: show remaining references and break if (hasRemainingRefs) { console.log( chalk.yellow( @@ -180,7 +231,9 @@ async function resolveReferences(options: ResolveOptions): Promise { ), ) console.log( - chalk.yellow(' Run the resolve command again to resolve them, or use --recursive'), + chalk.yellow( + ' Run the expand command again to expand them, or omit --shallow for full expansion', + ), ) if (verbose) { for (const ref of remainingRefs) { @@ -210,7 +263,7 @@ async function resolveReferences(options: ResolveOptions): Promise { } else if (!hasRemainingRefs && iteration > 1) { console.log( chalk.green( - `✓ Fully resolved all references in: ${filePath} (${iteration} iterations)`, + `✓ Fully expanded all references in: ${filePath} (${iteration} iterations)`, ), ) } @@ -224,7 +277,7 @@ async function resolveReferences(options: ResolveOptions): Promise { /** * Restore content by restoring original Liquid statements from HTML comments */ -async function restoreReferences(options: ResolveOptions): Promise { +async function restoreReferences(options: ExpandOptions): Promise { const { paths, verbose } = options const allowedTypes = getAllowedTypes(options) @@ -254,11 +307,11 @@ async function restoreReferences(options: ResolveOptions): Promise { if (hasEdits) { console.log( chalk.blue( - `ℹ️ Info: ${filePath} contains resolved references that will be preserved by updating data files`, + `${filePath} contains expanded references; any edits made will be preserved in data files`, ), ) if (!verbose) { - console.log(chalk.dim(' Use --verbose to see details of the edits')) + console.log(chalk.dim(' Use --verbose to see details of the edits')) } // Update data files with the edited content before restoring @@ -266,7 +319,8 @@ async function restoreReferences(options: ResolveOptions): Promise { // Automatically restore any updated data files back to liquid tags if (updatedDataFiles.length > 0) { - console.log(chalk.blue(' Restoring updated data files back to liquid tags...')) + if (verbose) + console.log(chalk.blue(' Restoring updated data files back to liquid tags...')) for (const dataFile of updatedDataFiles) { try { const dataContent = fs.readFileSync(dataFile, 'utf-8') @@ -274,12 +328,14 @@ async function restoreReferences(options: ResolveOptions): Promise { if (restoredDataContent !== dataContent) { fs.writeFileSync(dataFile, restoredDataContent, 'utf-8') if (verbose) { - console.log(chalk.green(` Restored: ${dataFile}`)) + const dataPath = convertFilePathToDataPath(dataFile) + console.log(chalk.green(` Restored: ${dataPath}`)) } } } catch (error) { if (verbose) { - console.log(chalk.yellow(` Could not restore ${dataFile}: ${error}`)) + const dataPath = convertFilePathToDataPath(dataFile) + console.log(chalk.yellow(` Could not restore ${dataPath}: ${error}`)) } } } @@ -293,7 +349,7 @@ async function restoreReferences(options: ResolveOptions): Promise { fs.writeFileSync(filePath, restoredContent, 'utf-8') console.log(chalk.green(`✓ Restored references in: ${filePath}`)) } else { - console.log(chalk.gray(`No resolved references found in: ${filePath}`)) + console.log(chalk.gray(`No expanded references found in: ${filePath}`)) } } catch (error: any) { console.error(chalk.red(`Error restoring ${filePath}: ${error.message}`)) @@ -302,9 +358,9 @@ async function restoreReferences(options: ResolveOptions): Promise { } /** - * Resolve all Liquid data references in file content + * Expand all Liquid data references in file content */ -async function resolveFileContent( +async function expandFileContent( content: string, filePath: string, verbose?: boolean, @@ -317,7 +373,7 @@ async function resolveFileContent( return content } - let resolvedContent = content + let expandedContent = content let offset = 0 for (const ref of references) { @@ -329,8 +385,8 @@ async function resolveFileContent( let replacement: string if (withMarkers) { - const commentStart = `` - const commentEnd = `` + const commentStart = `` + const commentEnd = `` replacement = `${commentStart}${resolvedValue}${commentEnd}` } else { replacement = resolvedValue @@ -339,33 +395,33 @@ async function resolveFileContent( const startPos = ref.startIndex + offset const endPos = ref.endIndex + offset - resolvedContent = - resolvedContent.substring(0, startPos) + replacement + resolvedContent.substring(endPos) + expandedContent = + expandedContent.substring(0, startPos) + replacement + expandedContent.substring(endPos) offset += replacement.length - originalText.length if (verbose) { - console.log(chalk.green(` Resolved: ${ref.type}s.${ref.path}`)) + console.log(chalk.green(` Expanded: ${ref.type}s.${ref.path}`)) } } else { if (verbose) { - console.log(chalk.yellow(` Warning: Could not resolve ${ref.type}s.${ref.path}`)) + console.log(chalk.yellow(` Warning: Could not expand ${ref.type}s.${ref.path}`)) } } } catch (error: any) { if (verbose) { - console.log(chalk.red(` Error resolving ${ref.type}s.${ref.path}: ${error.message}`)) + console.log(chalk.red(` Error expanding ${ref.type}s.${ref.path}: ${error.message}`)) } } } - // Note: Remaining reference detection is now handled in resolveReferences function for recursive mode + // Note: Remaining reference detection is now handled in expandReferences function for recursive mode - return resolvedContent + return expandedContent } /** - * Detect if resolved content has been edited by comparing with original data + * Detect if expanded content has been edited by comparing with original data */ async function detectContentEdits( content: string, @@ -375,7 +431,7 @@ async function detectContentEdits( let hasEdits = false let match - while ((match = RESOLVED_PATTERN.exec(content)) !== null) { + while ((match = EXPANDED_PATTERN.exec(content)) !== null) { const [, type, dataPath, resolvedContent] = match const refType = type as 'reusable' | 'variable' @@ -463,7 +519,7 @@ function restoreFileContent( verbose?: boolean, allowedTypes?: Array<'reusable' | 'variable'>, ): string { - return content.replace(RESOLVED_PATTERN, (match, type, dataPath) => { + return content.replace(EXPANDED_PATTERN, (match, type, dataPath) => { const refType = type as 'reusable' | 'variable' // Only restore if this type is allowed @@ -483,7 +539,7 @@ function restoreFileContent( } /** - * Update data files with content from resolved blocks + * Update data files with content from expanded blocks * Returns array of file paths that were updated */ function updateDataFiles( @@ -533,7 +589,7 @@ function updateDataFiles( } /** - * Extract data updates from resolved content blocks + * Extract data updates from expanded content blocks */ function extractDataUpdates( content: string, @@ -542,7 +598,7 @@ function extractDataUpdates( const updates: Array<{ type: 'reusable' | 'variable'; path: string; newContent: string }> = [] let match - while ((match = RESOLVED_PATTERN.exec(content)) !== null) { + while ((match = EXPANDED_PATTERN.exec(content)) !== null) { const [, type, dataPath, resolvedContent] = match const refType = type as 'reusable' | 'variable' @@ -632,7 +688,7 @@ function applyDataUpdates( fs.writeFileSync(targetPath, newContent) if (verbose) { - console.log(chalk.green(` Updated: ${targetPath}`)) + console.log(chalk.green(` Updated: ${type}s.${dataPath}`)) } } else { // For variables, update YAML structure @@ -669,13 +725,13 @@ function applyDataUpdates( // Write back to file fs.writeFileSync(targetPath, finalYaml) if (verbose) { - console.log(chalk.green(` Updated: ${targetPath}`)) + console.log(chalk.green(` Updated: ${type}s.${dataPath}`)) } } return targetPath } catch (error: any) { if (verbose) { - console.log(chalk.red(` Error updating ${targetPath}: ${error.message}`)) + console.log(chalk.red(` Error updating ${type}s.${dataPath}: ${error.message}`)) } return null } diff --git a/src/content-render/scripts/move-by-content-type.ts b/src/content-render/scripts/move-by-content-type.ts new file mode 100644 index 000000000000..2acc86c97f75 --- /dev/null +++ b/src/content-render/scripts/move-by-content-type.ts @@ -0,0 +1,642 @@ +/** + * @purpose Writer tool + * @description Move files to the relevant directory based on `contentType` frontmatter + */ + +import { program } from 'commander' +import fs from 'fs/promises' +import path from 'path' +import chalk from 'chalk' +import { execFileSync } from 'child_process' +import walkFiles from '@/workflows/walk-files.js' +import readFrontmatter from '@/frame/lib/read-frontmatter.js' +import { contentTypesEnum } from '@/frame/lib/frontmatter' + +const CONTENT_TYPES = contentTypesEnum.filter( + (type) => type !== 'homepage' && type !== 'other' && type !== 'landing', +) + +// The number of path segments at the product level (e.g., "content//..."). +// Used when determining whether a target directory is a deeper subdirectory. +const PRODUCT_LEVEL_PATH_SEGMENTS = 3 + +const contentTypeToDir = (contentType: string): string => { + return contentType === 'rai' ? 'responsible-use' : contentType +} + +const validContentTypeDirs = new Set(CONTENT_TYPES.map(contentTypeToDir)) + +// Helper: Should we skip this index.md file from processing? +function shouldSkipIndexFile(filePath: string): boolean { + const relativePath = path.relative(process.cwd(), filePath) + const parts = relativePath.split(path.sep) + const contentIndex = parts.indexOf('content') + + // Skip product-level index.md: content/product/index.md + if (parts.length === contentIndex + PRODUCT_LEVEL_PATH_SEGMENTS) return true + + // Skip content-type-level index.md that's already in place: content/product/content-type/index.md + if (parts.length === contentIndex + 4) { + const parentDir = parts[parts.length - 2] + if (validContentTypeDirs.has(parentDir)) return true + } + + return false +} + +// Helper: Calculate target directory for a file +function calculateTarget(filePath: string, contentType: string, productDir: string) { + const relativePath = path.relative(process.cwd(), filePath) + const parts = relativePath.split(path.sep) + const contentIndex = parts.indexOf('content') + const fileName = path.basename(filePath) + + // Determine target content-type directory + const targetContentType = contentTypeToDir(contentType) + + // Calculate target path + if (targetContentType === 'how-tos') { + // Preserve subdirectory structure for how-tos + const pathAfterProduct = parts.slice(contentIndex + 2, -1) + if (pathAfterProduct[0] === 'how-tos') { + // Already in how-tos, no change + return { targetDir: path.dirname(filePath), targetPath: filePath } + } else { + // Move to how-tos preserving structure + const targetDir = path.join(productDir, targetContentType, ...pathAfterProduct) + return { targetDir, targetPath: path.join(targetDir, fileName) } + } + } else { + // Flatten to content-type directory + const targetDir = path.join(productDir, targetContentType) + return { targetDir, targetPath: path.join(targetDir, fileName) } + } +} + +interface FileMove { + filePath: string + targetDir: string + targetPath: string + contentType: string +} + +program + .name('content-type-based-move') + .description('Reorganize content files into subdirectories based on their contentType property') + .argument('[paths...]', 'Content paths to process') + .action(async (paths: string[]) => { + // ==================== + // 1. GATHER FILES + // ==================== + const filesToProcess: string[] = [] + if (paths?.length > 0) { + for (const p of paths) { + const stats = await fs.stat(p) + if (stats.isDirectory()) { + filesToProcess.push(...(await walkFiles(p, ['.md']))) + } else if (p.endsWith('.md')) { + filesToProcess.push(p) + } + } + } else { + filesToProcess.push(...(await walkFiles(path.join(process.cwd(), 'content'), ['.md']))) + } + + console.log(chalk.white(`Processing ${filesToProcess.length} files...\n`)) + + // ==================== + // 2. ANALYZE & PLAN MOVES + // ==================== + console.log(chalk.white('Analyzing files...\n')) + + const filesToMove: FileMove[] = [] + const skipped: Array<{ file: string; reason: string }> = [] + const targetDirs = new Set() // Relative paths of all target directories + const subdirTargets = new Set() // Subdirectories receiving index.md files + const productDirs = new Set() + const productsWithRai = new Set() + + for (const filePath of filesToProcess) { + const relativePath = path.relative(process.cwd(), filePath) + + try { + // Skip certain index.md files + if (path.basename(filePath) === 'index.md' && shouldSkipIndexFile(filePath)) { + continue + } + + // Read and validate contentType + const fileContent = await fs.readFile(filePath, 'utf-8') + const { data } = readFrontmatter(fileContent) + + if (!data?.contentType) { + skipped.push({ file: relativePath, reason: 'No contentType property found' }) + console.log(chalk.yellow(`⚠ Skipping ${relativePath}: No contentType property`)) + continue + } + + const contentType = data.contentType as string + const parts = relativePath.split(path.sep) + const contentIndex = parts.indexOf('content') + + // Skip all landing pages - they should only be product-level index.md and don't move + if (contentType === 'landing') { + console.log(chalk.gray(`→ Skipping ${relativePath}: landing pages don't move`)) + continue + } + + // Validate contentType + if (!CONTENT_TYPES.includes(contentType as any)) { + skipped.push({ file: relativePath, reason: `Invalid contentType: ${contentType}` }) + console.log( + chalk.yellow(`⚠ Skipping ${relativePath}: Invalid contentType "${contentType}"`), + ) + continue + } + + // Get product directory + if (contentIndex === -1 || contentIndex + 1 >= parts.length) { + console.log( + chalk.yellow(`⚠ Skipping ${relativePath}: Cannot determine product directory`), + ) + continue + } + + const productName = parts[contentIndex + 1] + const productDir = path.join(process.cwd(), 'content', productName) + productDirs.add(productDir) + + if (contentType === 'rai') productsWithRai.add(productName) + + // Calculate target + const { targetDir, targetPath } = calculateTarget(filePath, contentType, productDir) + + // Skip if already in correct location + if (path.dirname(filePath) === targetDir) continue + + // Skip if target exists + try { + await fs.access(targetPath) + skipped.push({ file: relativePath, reason: 'Target already exists' }) + console.log(chalk.yellow(`⚠ Skipping ${relativePath}: Target file already exists`)) + continue + } catch { + // Good, doesn't exist + } + + // Track this move + filesToMove.push({ filePath, targetDir, targetPath, contentType }) + + const relativeTargetDir = path.relative(process.cwd(), targetDir) + targetDirs.add(relativeTargetDir) + + // Track subdirectories that will receive index.md files + if ( + path.basename(filePath) === 'index.md' && + relativeTargetDir.split(path.sep).length > PRODUCT_LEVEL_PATH_SEGMENTS + ) { + subdirTargets.add(relativeTargetDir) + } + } catch (error) { + if (error instanceof Error) { + console.error( + chalk.red(`✗ Error analyzing ${relativePath}: ${error.message}\n${error.stack}`), + ) + } else { + console.error(chalk.red(`✗ Error analyzing ${relativePath}: ${String(error)}`)) + } + skipped.push({ file: relativePath, reason: `Error: ${error}` }) + } + } + + // ==================== + // 3. ENSURE STANDARD DIRECTORIES + // ==================== + console.log(chalk.white('Ensuring standard content-type directories exist...\n')) + + // Add standard content-type directories for each affected product + if (paths?.length > 0) { + for (const p of paths) { + const fullPath = path.resolve(process.cwd(), p) + const relativePath = path.relative(process.cwd(), fullPath) + const parts = relativePath.split(path.sep) + const contentIndex = parts.indexOf('content') + + if (contentIndex !== -1 && contentIndex + 1 < parts.length) { + const productName = parts[contentIndex + 1] + + for (const ct of CONTENT_TYPES.filter((t) => t !== 'rai' && t !== 'landing')) { + targetDirs.add(path.join('content', productName, contentTypeToDir(ct))) + } + + if (productsWithRai.has(productName)) { + targetDirs.add(path.join('content', productName, 'responsible-use')) + } + } + } + } + + // ==================== + // 4. CREATE PLACEHOLDERS + // ==================== + console.log(chalk.white('Creating placeholder index.md files...\n')) + + const newPlaceholders: string[] = [] + const titleMap: Record = { + 'get-started': 'Get started', + concepts: 'Concepts', + 'how-tos': 'How-tos', + reference: 'Reference', + tutorials: 'Tutorials', + 'responsible-use': 'Responsible use', + } + + for (const dirPath of targetDirs) { + const absoluteDirPath = path.join(process.cwd(), dirPath) + const indexPath = path.join(absoluteDirPath, 'index.md') + + try { + await fs.access(indexPath) + console.log(chalk.gray(`- Skipping ${dirPath}/index.md (already exists)`)) + } catch { + // Only create placeholders for top-level content-type directories (not subdirectories) + if (dirPath.split(path.sep).length > PRODUCT_LEVEL_PATH_SEGMENTS) continue + + // Skip if an index.md will be moved here + if (subdirTargets.has(dirPath)) { + console.log(chalk.gray(`- Skipping ${dirPath}/index.md (will be moved)`)) + continue + } + + const contentTypeName = path.basename(dirPath) + const title = titleMap[contentTypeName] || contentTypeName + + // Determine the correct contentType for this placeholder + // Map directory name back to contentType enum value + const placeholderContentType = + contentTypeName === 'responsible-use' ? 'rai' : contentTypeName + + const content = `--- +title: ${title} +versions: + fpt: '*' + ghec: '*' + ghes: '*' +contentType: ${placeholderContentType} +--- + +` + + await fs.mkdir(absoluteDirPath, { recursive: true }) + await fs.writeFile(indexPath, content, 'utf-8') + newPlaceholders.push(indexPath) + + console.log(chalk.green(`✓ Created ${dirPath}/index.md`)) + } + } + + // ==================== + // 5. GENERATE INTROS + // ==================== + if (newPlaceholders.length > 0) { + console.log(chalk.white('\nGenerating intros for placeholder files...\n')) + + for (const placeholderPath of newPlaceholders) { + try { + const fileContent = await fs.readFile(placeholderPath, 'utf-8') + const { data } = readFrontmatter(fileContent) + + if (data?.intro) continue + + const relativePath = path.relative(process.cwd(), placeholderPath) + console.log(chalk.gray(`Generating intro for ${relativePath}...`)) + + execFileSync( + 'npm', + ['run', 'ai-tools', '--', '--prompt', 'intro', '--files', relativePath, '--write'], + { + cwd: process.cwd(), + stdio: 'inherit', + }, + ) + + console.log(chalk.green(`✓ Generated intro for ${relativePath}`)) + } catch (error) { + if (error instanceof Error) { + console.error( + chalk.yellow( + `⚠ Could not generate intro for ${placeholderPath}: ${error.message}\n${error.stack}`, + ), + ) + } else { + console.error( + chalk.yellow(`⚠ Could not generate intro for ${placeholderPath}: ${String(error)}`), + ) + } + } + } + } + + // ==================== + // 6. MOVE FILES + // ==================== + console.log(chalk.white('\nMoving files...\n')) + + const moved: Array<{ file: string; from: string; to: string }> = [] + + // Categorize files by type for correct move order + const regularFiles = filesToMove.filter((f) => path.basename(f.filePath) !== 'index.md') + const topLevelIndexFiles = filesToMove.filter((f) => { + if (path.basename(f.filePath) !== 'index.md') return false + return ( + path.relative(process.cwd(), f.targetDir).split(path.sep).length === + PRODUCT_LEVEL_PATH_SEGMENTS + ) + }) + const subdirIndexFiles = filesToMove.filter((f) => { + if (path.basename(f.filePath) !== 'index.md') return false + return ( + path.relative(process.cwd(), f.targetDir).split(path.sep).length > + PRODUCT_LEVEL_PATH_SEGMENTS + ) + }) + + // Move subdirectory index files first (copy only, delete later) + const indexFilesToDeleteLater: string[] = [] + for (const file of subdirIndexFiles) { + try { + await fs.mkdir(file.targetDir, { recursive: true }) + + const content = await fs.readFile(file.filePath, 'utf-8') + const { data, content: body } = readFrontmatter(content) + // Clear children array because paths will be invalid in the new content-type directory structure + if (data?.children) data.children = [] + + await fs.writeFile( + file.targetPath, + readFrontmatter.stringify(body || '', data || {}), + 'utf-8', + ) + indexFilesToDeleteLater.push(file.filePath) + + moved.push({ + file: path.relative(process.cwd(), file.filePath), + from: path.relative(process.cwd(), file.filePath), + to: path.relative(process.cwd(), file.targetPath), + }) + + console.log(chalk.green(`✓ Copied ${path.relative(process.cwd(), file.filePath)}`)) + } catch (error) { + skipped.push({ + file: path.relative(process.cwd(), file.filePath), + reason: `Error: ${error}`, + }) + console.log( + chalk.red(`✗ Error copying ${path.relative(process.cwd(), file.filePath)}: ${error}`), + ) + } + } + + // Move regular files + for (const file of regularFiles) { + try { + await fs.mkdir(file.targetDir, { recursive: true }) + + const relativeFilePath = path.relative(process.cwd(), file.filePath) + const relativeTargetPath = path.relative(process.cwd(), file.targetPath) + + execFileSync( + 'npm', + ['run', 'move-content', '--', relativeFilePath, relativeTargetPath, '--no-git'], + { + cwd: process.cwd(), + stdio: 'inherit', + }, + ) + + moved.push({ file: relativeFilePath, from: relativeFilePath, to: relativeTargetPath }) + console.log(chalk.green(`✓ Moved ${relativeFilePath}`)) + } catch (error) { + skipped.push({ + file: path.relative(process.cwd(), file.filePath), + reason: `Error: ${error}`, + }) + console.log( + chalk.red(`✗ Error moving ${path.relative(process.cwd(), file.filePath)}: ${error}`), + ) + } + } + + // Delete source subdirectory index files + for (const sourcePath of indexFilesToDeleteLater) { + try { + await fs.unlink(sourcePath) + console.log(chalk.gray(`✓ Deleted source ${path.relative(process.cwd(), sourcePath)}`)) + } catch (error) { + console.log(chalk.yellow(`⚠ Could not delete ${sourcePath}: ${error}`)) + } + } + + // Move top-level index files + for (const file of topLevelIndexFiles) { + try { + await fs.mkdir(file.targetDir, { recursive: true }) + await fs.copyFile(file.filePath, file.targetPath) + await fs.unlink(file.filePath) + + moved.push({ + file: path.relative(process.cwd(), file.filePath), + from: path.relative(process.cwd(), file.filePath), + to: path.relative(process.cwd(), file.targetPath), + }) + + console.log(chalk.green(`✓ Moved ${path.relative(process.cwd(), file.filePath)}`)) + } catch (error) { + skipped.push({ + file: path.relative(process.cwd(), file.filePath), + reason: `Error: ${error}`, + }) + console.log( + chalk.red(`✗ Error moving ${path.relative(process.cwd(), file.filePath)}: ${error}`), + ) + } + } + + // ==================== + // 7. CLEANUP & UPDATE + // ==================== + console.log( + chalk.white('\nCleaning up old directories and updating parent index.md files...\n'), + ) + + const deletedByProduct = new Map() + + for (const productDir of productDirs) { + const productName = path.basename(productDir) + + try { + const entries = await fs.readdir(productDir, { withFileTypes: true }) + + for (const entry of entries) { + if (entry.isDirectory() && !validContentTypeDirs.has(entry.name)) { + const oldDirPath = path.join(productDir, entry.name) + + try { + await fs.rm(oldDirPath, { recursive: true, force: true }) + console.log( + chalk.gray(`✓ Deleted old directory: ${path.relative(process.cwd(), oldDirPath)}`), + ) + + if (!deletedByProduct.has(productName)) deletedByProduct.set(productName, []) + deletedByProduct.get(productName)!.push(`/${productName}/${entry.name}`) + } catch (error) { + console.log(chalk.yellow(`⚠ Could not delete ${oldDirPath}: ${error}`)) + } + } + } + } catch (error) { + console.log(chalk.yellow(`⚠ Could not read product directory ${productDir}: ${error}`)) + } + + // Update product index.md + const productIndexPath = path.join(productDir, 'index.md') + try { + const content = await fs.readFile(productIndexPath, 'utf-8') + const { data, content: body } = readFrontmatter(content) + + if (data) { + let updated = false + + // Build children array + const productRelativePath = path.relative(process.cwd(), productDir) + const newChildren: string[] = [] + for (const ct of CONTENT_TYPES.map(contentTypeToDir)) { + const dirPath = path.join(productRelativePath, ct) + if (targetDirs.has(dirPath)) newChildren.push(`/${ct}`) + } + + if (newChildren.length > 0) { + data.children = newChildren + updated = true + } + + // Add redirects for deleted directories + const deletedPaths = deletedByProduct.get(productName) || [] + if (deletedPaths.length > 0) { + if (!data.redirect_from) data.redirect_from = [] + else if (!Array.isArray(data.redirect_from)) data.redirect_from = [data.redirect_from] + + for (const deletedPath of deletedPaths) { + if (!data.redirect_from.includes(deletedPath)) { + data.redirect_from.push(deletedPath) + } + } + updated = true + } + + if (updated) { + await fs.writeFile(productIndexPath, readFrontmatter.stringify(body, data), 'utf-8') + const changes = [] + if (newChildren.length > 0) changes.push(`${newChildren.length} children`) + if (deletedPaths.length > 0) changes.push(`${deletedPaths.length} redirects`) + console.log( + chalk.green( + `✓ Updated ${path.relative(process.cwd(), productIndexPath)} (${changes.join(', ')})`, + ), + ) + } + } + } catch (error) { + console.log(chalk.yellow(`⚠ Could not update ${productIndexPath}: ${error}`)) + } + } + + // ==================== + // 8. SORT CHILDREN ARRAYS + // ==================== + console.log(chalk.white('\nSorting children arrays...\n')) + + for (const dirPath of targetDirs) { + const absoluteDirPath = path.join(process.cwd(), dirPath) + const indexPath = path.join(absoluteDirPath, 'index.md') + + try { + const content = await fs.readFile(indexPath, 'utf-8') + const { data, content: body } = readFrontmatter(content) + + if (!data) continue + + // For how-tos, build children from subdirectories + if (path.basename(dirPath) === 'how-tos') { + const entries = await fs.readdir(absoluteDirPath, { withFileTypes: true }) + const subdirs = entries + .filter((e) => e.isDirectory()) + .map((e) => `/${e.name}`) + .sort() + + if (subdirs.length > 0) { + data.children = subdirs + await fs.writeFile(indexPath, readFrontmatter.stringify(body, data), 'utf-8') + console.log( + chalk.green( + `✓ Added children to ${path.relative(process.cwd(), indexPath)} (${subdirs.length} subdirectories)`, + ), + ) + } + } + // For others, sort with about-* first + else if (data.children && Array.isArray(data.children) && data.children.length > 0) { + const sorted = [...data.children].sort((a, b) => { + const aBasename = path.basename(a) + const bBasename = path.basename(b) + const aIsAbout = aBasename.startsWith('about-') + const bIsAbout = bBasename.startsWith('about-') + + if (aIsAbout && !bIsAbout) return -1 + if (!aIsAbout && bIsAbout) return 1 + return aBasename.localeCompare(bBasename) + }) + + if (JSON.stringify(sorted) !== JSON.stringify(data.children)) { + data.children = sorted + await fs.writeFile(indexPath, readFrontmatter.stringify(body, data), 'utf-8') + console.log( + chalk.green(`✓ Sorted children in ${path.relative(process.cwd(), indexPath)}`), + ) + } + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + console.log(chalk.yellow(`⚠ Could not update ${indexPath}: ${error}`)) + } + } + } + + // ==================== + // 9. SUMMARY + // ==================== + console.log(chalk.white(`\n${'='.repeat(60)}`)) + console.log(chalk.white('Summary:')) + console.log(chalk.white(` Moved: ${moved.length} files`)) + console.log(chalk.white(` Skipped: ${skipped.length} files`)) + + if (newPlaceholders.length > 0) { + console.log( + chalk.cyan( + `\nNote: ${newPlaceholders.length} placeholder index.md files were created with`, + ), + ) + console.log(chalk.cyan(`AI-generated intros. Please review before committing.`)) + } + + console.log(chalk.blue('='.repeat(60))) + + if (skipped.length > 0) { + console.log(chalk.yellow('\nSkipped files:')) + for (const skip of skipped) { + console.log(chalk.gray(` ${skip.file}: ${skip.reason}`)) + } + } + }) + +program.parse() diff --git a/src/content-render/tests/resolve-liquid.ts b/src/content-render/tests/liquid-tags.ts similarity index 72% rename from src/content-render/tests/resolve-liquid.ts rename to src/content-render/tests/liquid-tags.ts index 70e23a627ad6..5505f28c43c5 100644 --- a/src/content-render/tests/resolve-liquid.ts +++ b/src/content-render/tests/liquid-tags.ts @@ -6,7 +6,7 @@ import { execSync } from 'child_process' const rootDir = path.join(__dirname, '../../..') const testContentDir = path.join(rootDir, 'content/test-integration') -describe('resolve-liquid script integration tests', () => { +describe('liquid-tags script integration tests', () => { vi.setConfig({ testTimeout: 60 * 1000 }) beforeEach(async () => { @@ -20,12 +20,12 @@ describe('resolve-liquid script integration tests', () => { }) // Helper function to run script commands - async function runResolveScript(args: string): Promise<{ output: string; exitCode: number }> { + async function runScript(args: string): Promise<{ output: string; exitCode: number }> { let output = '' let exitCode = 0 try { - output = execSync(`tsx src/content-render/scripts/resolve-liquid.ts ${args}`, { + output = execSync(`tsx src/content-render/scripts/liquid-tags.ts ${args}`, { encoding: 'utf8', cwd: rootDir, stdio: 'pipe', @@ -39,7 +39,7 @@ describe('resolve-liquid script integration tests', () => { return { output, exitCode } } - test('resolve command should complete successfully with basic content', async () => { + test('expand command should complete successfully with basic content', async () => { // Create a test file with liquid reference const testFile = path.join(testContentDir, 'basic-test.md') const testContent = `--- @@ -51,16 +51,16 @@ This uses {% data variables.product.prodname_dotcom %} in content. await fs.writeFile(testFile, testContent) - const { output, exitCode } = await runResolveScript(`resolve --paths "${testFile}"`) + const { output, exitCode } = await runScript(`expand --paths "${testFile}"`) // Should complete without error expect(exitCode, `Script failed with output: ${output}`).toBe(0) expect(output.length).toBeGreaterThan(0) // Check that the file was modified - const resolvedContent = await fs.readFile(testFile, 'utf8') - expect(resolvedContent).not.toBe(testContent) - expect(resolvedContent).toContain('GitHub') // Should resolve to actual fixture value + const expandedContent = await fs.readFile(testFile, 'utf8') + expect(expandedContent).not.toBe(testContent) + expect(expandedContent).toContain('GitHub') // Should expand to actual fixture value }) test('restore command should complete successfully', async () => { @@ -74,11 +74,11 @@ This uses {% data variables.product.prodname_dotcom %} in content. await fs.writeFile(testFile, originalContent) - // First resolve - await runResolveScript(`resolve --paths "${testFile}"`) + // First expand + await runScript(`expand --paths "${testFile}"`) // Then restore - const { output, exitCode } = await runResolveScript(`restore --paths "${testFile}"`) + const { output, exitCode } = await runScript(`restore --paths "${testFile}"`) expect(exitCode, `Restore script failed with output: ${output}`).toBe(0) expect(output.length).toBeGreaterThan(0) @@ -90,7 +90,7 @@ This uses {% data variables.product.prodname_dotcom %} in content. }) test('help command should display usage information', async () => { - const { output, exitCode } = await runResolveScript('resolve --help') + const { output, exitCode } = await runScript('expand --help') expect(exitCode, `Help command failed with output: ${output}`).toBe(0) expect(output).toMatch(/resolve|usage|help|command/i) diff --git a/src/fixtures/fixtures/content/codespaces/guides.md b/src/fixtures/fixtures/content/codespaces/guides.md new file mode 100644 index 000000000000..e6d2b6800bb4 --- /dev/null +++ b/src/fixtures/fixtures/content/codespaces/guides.md @@ -0,0 +1,14 @@ +--- +title: Guides for GitHub Codespaces +intro: Learn how to make the most of GitHub Codespaces. +layout: product-guides +versions: + fpt: '*' +topics: + - Codespaces + - Developer +includeGuides: + - /get-started/start-your-journey/hello-world +learningTracks: + - foo_bar +--- diff --git a/src/fixtures/fixtures/content/codespaces/index.md b/src/fixtures/fixtures/content/codespaces/index.md new file mode 100644 index 000000000000..d9e241ea294a --- /dev/null +++ b/src/fixtures/fixtures/content/codespaces/index.md @@ -0,0 +1,10 @@ +--- +title: GitHub Codespaces documentation +intro: Develop in a codespace. +versions: + fpt: '*' +children: + - /guides +topics: + - Codespaces +--- diff --git a/src/fixtures/fixtures/content/index.md b/src/fixtures/fixtures/content/index.md index ed523bc00b95..c4fda9c57663 100644 --- a/src/fixtures/fixtures/content/index.md +++ b/src/fixtures/fixtures/content/index.md @@ -29,6 +29,7 @@ children: - early-access - pages - code-security + - codespaces - actions - rest - webhooks diff --git a/src/fixtures/fixtures/data/learning-tracks/codespaces.yml b/src/fixtures/fixtures/data/learning-tracks/codespaces.yml new file mode 100644 index 000000000000..78343b79c7ba --- /dev/null +++ b/src/fixtures/fixtures/data/learning-tracks/codespaces.yml @@ -0,0 +1,6 @@ +# Codespaces learning tracks +foo_bar: + title: Sample learning track + description: A sample track for testing + guides: + - /get-started/start-your-journey/hello-world diff --git a/src/github-apps/lib/index.ts b/src/github-apps/lib/index.ts index 0aab639fb5c1..98d840ef0a96 100644 --- a/src/github-apps/lib/index.ts +++ b/src/github-apps/lib/index.ts @@ -12,6 +12,7 @@ interface AppsConfig { // Note: Using 'any' for AppsData to maintain compatibility with existing consumers that expect different shapes type AppsData = any +const DEBUG = process.env.RUNNER_DEBUG === '1' || process.env.DEBUG === '1' const ENABLED_APPS_DIR = 'src/github-apps/data' const githubAppsData = new Map>() @@ -29,6 +30,12 @@ export async function getAppsData( docsVersion: string, apiVersion?: string, ): Promise { + if (DEBUG) { + console.log( + `[DEBUG] getAppsData: ROOT=${process.env.ROOT || '(not set)'}, path=${ENABLED_APPS_DIR}`, + ) + } + const pageTypeMap = githubAppsData.get(pageType)! const filename = `${pageType}.json` const openApiVersion = getOpenApiVersion(docsVersion) + (apiVersion ? `-${apiVersion}` : '')