diff --git a/Dockerfile.otel b/Dockerfile.otel new file mode 100644 index 0000000..a7abdfc --- /dev/null +++ b/Dockerfile.otel @@ -0,0 +1,4 @@ +FROM otel/opentelemetry-collector-contrib:0.136.0 +COPY src/lib/otel/receivers_config.yml /etc/otel/receivers_config.yml +COPY src/lib/otel/production_config.yml /etc/otel/production_config.yml +CMD ["--config", "/etc/otel/production_config.yml", "--config", "/etc/otel/receivers_config.yml"] \ No newline at end of file diff --git a/deployment/development/docker-compose.yml b/deployment/development/docker-compose.yml index acccd8a..588a39e 100644 --- a/deployment/development/docker-compose.yml +++ b/deployment/development/docker-compose.yml @@ -37,3 +37,19 @@ services: DATABASE_URL: "postgresql://db-user:1234@db:5432/dev-buildengine?schema=public" # MUST be included (the path the application will be accessed on) and MUST NOT have a trailing slash ORIGIN: "http://localhost:8443" + + otel: + image: otel/opentelemetry-collector-contrib:0.136.0 + ports: + - "6317:6317" # gRPC + - "6318:6318" # HTTP + - "55679:55679" # UI + volumes: + - ./src/lib/otel/development_config.yml:/etc/development_config.yml + - ./src/lib/otel/receivers_config.yml:/etc/receivers_config.yml + command: ["--config", "/etc/development_config.yml", "--config", "/etc/receivers_config.yml"] + + jaeger: + image: cr.jaegertracing.io/jaegertracing/jaeger:latest + ports: + - "16686:16686" diff --git a/package-lock.json b/package-lock.json index 3929dfe..1d22a78 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@aws-sdk/client-s3": "^3.907.0", "@aws-sdk/client-sts": "^3.907.0", "@prisma/client": "^6.15.0", + "bullmq-otel": "^1.1.1", "prisma": "^6.15.0", "s3-sync-client": "^4.3.1" }, @@ -26,6 +27,16 @@ "@eslint/js": "^9.18.0", "@hono/node-server": "^1.19.5", "@iconify/svelte": "^5.0.2", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-grpc": "^0.211.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.211.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.211.0", + "@opentelemetry/instrumentation": "^0.211.0", + "@opentelemetry/resources": "^2.5.0", + "@opentelemetry/sdk-logs": "^0.211.0", + "@opentelemetry/sdk-metrics": "^2.5.0", + "@opentelemetry/sdk-node": "^0.211.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@sveltejs/adapter-node": "^5.2.12", "@sveltejs/kit": "^2.22.0", "@sveltejs/vite-plugin-svelte": "^6.0.0", @@ -3866,6 +3877,39 @@ "license": "MIT", "optional": true }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.3", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@hapi/hoek": { "version": "9.3.0", "resolved": "https://registry.npmjs.org/@hapi/hoek/-/hoek-9.3.0.tgz", @@ -3977,7 +4021,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.4.0.tgz", "integrity": "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ==", - "dev": true, "license": "MIT" }, "node_modules/@isaacs/fs-minipass": { @@ -4043,6 +4086,17 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz", @@ -4050,7 +4104,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4064,7 +4117,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4078,7 +4130,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4092,7 +4143,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4106,7 +4156,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4120,7 +4169,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -4165,6 +4213,570 @@ "node": ">= 8" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.211.0.tgz", + "integrity": "sha512-swFdZq8MCdmdR22jTVGQDhwqDzcI4M10nhjXkLr1EsIzXgZBqm4ZlmmcWsg3TSNf+3mzgOiqveXmBLZuDi2Lgg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/configuration": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/configuration/-/configuration-0.211.0.tgz", + "integrity": "sha512-PNsCkzsYQKyv8wiUIsH+loC4RYyblOaDnVASBtKS22hK55ToWs2UP6IsrcfSWWn54wWTvVe2gnfwz67Pvrxf2Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "yaml": "^2.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.9.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.5.0.tgz", + "integrity": "sha512-uOXpVX0ZjO7heSVjhheW2XEPrhQAWr2BScDPoZ9UDycl5iuHG+Usyc3AIfG6kZeC1GyLpMInpQ6X5+9n69yOFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.0.tgz", + "integrity": "sha512-ka4H8OM6+DlUhSAZpONu0cPBtPPTQKxbxVzC4CzVx5+K4JnroJVBtDzLAMx4/3CDTJXRvVFhpFjtl4SaiTNoyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.211.0.tgz", + "integrity": "sha512-UhOoWENNqyaAMP/dL1YXLkXt6ZBtovkDDs1p4rxto9YwJX1+wMjwg+Obfyg2kwpcMoaiIFT3KQIcLNW8nNGNfQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/sdk-logs": "0.211.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.211.0.tgz", + "integrity": "sha512-c118Awf1kZirHkqxdcF+rF5qqWwNjJh+BB1CmQvN9AQHC/DUIldy6dIkJn3EKlQnQ3HmuNRKc/nHHt5IusN7mA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/sdk-logs": "0.211.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-proto": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-proto/-/exporter-logs-otlp-proto-0.211.0.tgz", + "integrity": "sha512-kMvfKMtY5vJDXeLnwhrZMEwhZ2PN8sROXmzacFU/Fnl4Z79CMrOaL7OE+5X3SObRYlDUa7zVqaXp9ZetYCxfDQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-logs": "0.211.0", + "@opentelemetry/sdk-trace-base": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.211.0.tgz", + "integrity": "sha512-D/U3G8L4PzZp8ot5hX9wpgbTymgtLZCiwR7heMe4LsbGV4OdctS1nfyvaQHLT6CiGZ6FjKc1Vk9s6kbo9SWLXQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.211.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-metrics": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.211.0.tgz", + "integrity": "sha512-lfHXElPAoDSPpPO59DJdN5FLUnwi1wxluLTWQDayqrSPfWRnluzxRhD+g7rF8wbj1qCz0sdqABl//ug1IZyWvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-metrics": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.211.0.tgz", + "integrity": "sha512-61iNbffEpyZv/abHaz3BQM3zUtA2kVIDBM+0dS9RK68ML0QFLRGYa50xVMn2PYMToyfszEPEgFC3ypGae2z8FA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.211.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-metrics": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-prometheus": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.211.0.tgz", + "integrity": "sha512-cD0WleEL3TPqJbvxwz5MVdVJ82H8jl8mvMad4bNU24cB5SH2mRW5aMLDTuV4614ll46R//R3RMmci26mc2L99g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-metrics": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.211.0.tgz", + "integrity": "sha512-eFwx4Gvu6LaEiE1rOd4ypgAiWEdZu7Qzm2QNN2nJqPW1XDeAVH1eNwVcVQl+QK9HR/JCDZ78PZgD7xD/DBDqbw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.211.0.tgz", + "integrity": "sha512-F1Rv3JeMkgS//xdVjbQMrI3+26e5SXC7vXA6trx8SWEA0OUhw4JHB+qeHtH0fJn46eFItrYbL5m8j4qi9Sfaxw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-proto": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-proto/-/exporter-trace-otlp-proto-0.211.0.tgz", + "integrity": "sha512-DkjXwbPiqpcPlycUojzG2RmR0/SIK8Gi9qWO9znNvSqgzrnAIE9x2n6yPfpZ+kWHZGafvsvA1lVXucTyyQa5Kg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-zipkin": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-zipkin/-/exporter-zipkin-2.5.0.tgz", + "integrity": "sha512-bk9VJgFgUAzkZzU8ZyXBSWiUGLOM3mZEgKJ1+jsZclhRnAoDNf+YBdq+G9R3cP0+TKjjWad+vVrY/bE/vRR9lA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.0.0" + } + }, + "node_modules/@opentelemetry/instrumentation": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.211.0.tgz", + "integrity": "sha512-h0nrZEC/zvI994nhg7EgQ8URIHt0uDTwN90r3qQUdZORS455bbx+YebnGeEuFghUT0HlJSrLF4iHw67f+odY+Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "import-in-the-middle": "^2.0.0", + "require-in-the-middle": "^8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.211.0.tgz", + "integrity": "sha512-bp1+63V8WPV+bRI9EQG6E9YID1LIHYSZVbp7f+44g9tRzCq+rtw/o4fpL5PC31adcUsFiz/oN0MdLISSrZDdrg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-transformer": "0.211.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.211.0.tgz", + "integrity": "sha512-mR5X+N4SuphJeb7/K7y0JNMC8N1mB6gEtjyTLv+TSAhl0ZxNQzpSKP8S5Opk90fhAqVYD4R0SQSAirEBlH1KSA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.7.1", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/otlp-exporter-base": "0.211.0", + "@opentelemetry/otlp-transformer": "0.211.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.211.0.tgz", + "integrity": "sha512-julhCJ9dXwkOg9svuuYqqjXLhVaUgyUvO2hWbTxwjvLXX2rG3VtAaB0SzxMnGTuoCZizBT7Xqqm2V7+ggrfCXA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-logs": "0.211.0", + "@opentelemetry/sdk-metrics": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0", + "protobufjs": "8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/protobufjs": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.0.0.tgz", + "integrity": "sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@opentelemetry/propagator-b3": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-b3/-/propagator-b3-2.5.0.tgz", + "integrity": "sha512-g10m4KD73RjHrSvUge+sUxUl8m4VlgnGc6OKvo68a4uMfaLjdFU+AULfvMQE/APq38k92oGUxEzBsAZ8RN/YHg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/propagator-jaeger": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/propagator-jaeger/-/propagator-jaeger-2.5.0.tgz", + "integrity": "sha512-t70ErZCncAR/zz5AcGkL0TF25mJiK1FfDPEQCgreyAHZ+mRJ/bNUiCnImIBDlP3mSDXy6N09DbUEKq0ktW98Hg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.0.tgz", + "integrity": "sha512-F8W52ApePshpoSrfsSk1H2yJn9aKjCrbpQF1M9Qii0GHzbfVeFUB+rc3X4aggyZD8x9Gu3Slua+s6krmq6Dt8g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.211.0.tgz", + "integrity": "sha512-O5nPwzgg2JHzo59kpQTPUOTzFi0Nv5LxryG27QoXBciX3zWM3z83g+SNOHhiQVYRWFSxoWn1JM2TGD5iNjOwdA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.5.0.tgz", + "integrity": "sha512-BeJLtU+f5Gf905cJX9vXFQorAr6TAfK3SPvTFqP+scfIpDQEJfRaGJWta7sJgP+m4dNtBf9y3yvBKVAZZtJQVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-node": { + "version": "0.211.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-node/-/sdk-node-0.211.0.tgz", + "integrity": "sha512-+s1eGjoqmPCMptNxcJJD4IxbWJKNLOQFNKhpwkzi2gLkEbCj6LzSHJNhPcLeBrBlBLtlSpibM+FuS7fjZ8SSFQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.211.0", + "@opentelemetry/configuration": "0.211.0", + "@opentelemetry/context-async-hooks": "2.5.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/exporter-logs-otlp-grpc": "0.211.0", + "@opentelemetry/exporter-logs-otlp-http": "0.211.0", + "@opentelemetry/exporter-logs-otlp-proto": "0.211.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "0.211.0", + "@opentelemetry/exporter-metrics-otlp-http": "0.211.0", + "@opentelemetry/exporter-metrics-otlp-proto": "0.211.0", + "@opentelemetry/exporter-prometheus": "0.211.0", + "@opentelemetry/exporter-trace-otlp-grpc": "0.211.0", + "@opentelemetry/exporter-trace-otlp-http": "0.211.0", + "@opentelemetry/exporter-trace-otlp-proto": "0.211.0", + "@opentelemetry/exporter-zipkin": "2.5.0", + "@opentelemetry/instrumentation": "0.211.0", + "@opentelemetry/propagator-b3": "2.5.0", + "@opentelemetry/propagator-jaeger": "2.5.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/sdk-logs": "0.211.0", + "@opentelemetry/sdk-metrics": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0", + "@opentelemetry/sdk-trace-node": "2.5.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.0.tgz", + "integrity": "sha512-VzRf8LzotASEyNDUxTdaJ9IRJ1/h692WyArDBInf5puLCjxbICD6XkHgpuudis56EndyS7LYFmtTMny6UABNdQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.0", + "@opentelemetry/resources": "2.5.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.5.0.tgz", + "integrity": "sha512-O6N/ejzburFm2C84aKNrwJVPpt6HSTSq8T0ZUMq3xT2XmqT4cwxUItcL5UWGThYuq8RTcbH8u1sfj6dmRci0Ow==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.5.0", + "@opentelemetry/core": "2.5.0", + "@opentelemetry/sdk-trace-base": "2.5.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.39.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.39.0.tgz", + "integrity": "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgr/core": { "version": "0.2.9", "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.2.9.tgz", @@ -4272,6 +4884,80 @@ "@prisma/debug": "6.15.0" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/@rollup/plugin-commonjs": { "version": "28.0.6", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-28.0.6.tgz", @@ -6514,6 +7200,16 @@ "node": ">=0.4.0" } }, + "node_modules/acorn-import-attributes": { + "version": "1.9.5", + "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz", + "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^8" + } + }, "node_modules/acorn-jsx": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", @@ -6541,6 +7237,16 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -6800,7 +7506,6 @@ "version": "5.61.0", "resolved": "https://registry.npmjs.org/bullmq/-/bullmq-5.61.0.tgz", "integrity": "sha512-khaTjc1JnzaYFl4FrUtsSsqugAW/urRrcZ9Q0ZE+REAw8W+gkHFqxbGlutOu6q7j7n91wibVaaNlOUMdiEvoSQ==", - "dev": true, "license": "MIT", "dependencies": { "cron-parser": "^4.9.0", @@ -6812,6 +7517,23 @@ "uuid": "^11.1.0" } }, + "node_modules/bullmq-otel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/bullmq-otel/-/bullmq-otel-1.1.1.tgz", + "integrity": "sha512-AySECCquHi/wE4kQUVVUr55I0CNVJDD7cisATTkUX21rBgphOf3qraoqCIllsxAQ32OJ0FtaTvZOrV2APrNgoA==", + "license": "MIT", + "dependencies": { + "@opentelemetry/api": "^1.9.0", + "bullmq": "^5.22.0", + "tslib": "^2.8.0" + } + }, + "node_modules/bullmq-otel/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/c12": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/c12/-/c12-3.1.0.tgz", @@ -6965,6 +7687,13 @@ "consola": "^3.2.3" } }, + "node_modules/cjs-module-lexer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-2.2.0.tgz", + "integrity": "sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==", + "dev": true, + "license": "MIT" + }, "node_modules/class-validator": { "version": "0.14.2", "resolved": "https://registry.npmjs.org/class-validator/-/class-validator-0.14.2.tgz", @@ -6978,6 +7707,21 @@ "validator": "^13.9.0" } }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -6992,7 +7736,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=0.10.0" @@ -7061,7 +7804,6 @@ "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==", - "dev": true, "license": "MIT", "dependencies": { "luxon": "^3.2.1" @@ -7174,7 +7916,6 @@ "version": "4.4.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", - "dev": true, "license": "MIT", "dependencies": { "ms": "^2.1.3" @@ -7260,7 +8001,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=0.10" @@ -7276,7 +8016,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -7363,6 +8103,13 @@ "node": ">=0.10.0" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, "node_modules/empathic": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/empathic/-/empathic-2.0.0.tgz", @@ -7595,6 +8342,16 @@ "esbuild": "*" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -8273,6 +9030,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", @@ -8562,6 +9329,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/import-in-the-middle": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-2.0.6.tgz", + "integrity": "sha512-3vZV3jX0XRFW3EJDTwzWoZa+RH1b8eTTx6YOCjglrLyPuepwoBti1k3L2dKwdCUrnVEfc5CuRuGstaC/uQJJaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "acorn": "^8.15.0", + "acorn-import-attributes": "^1.9.5", + "cjs-module-lexer": "^2.2.0", + "module-details-from-path": "^1.0.4" + } + }, "node_modules/imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -8591,7 +9371,6 @@ "version": "5.8.2", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.8.2.tgz", "integrity": "sha512-C6uC+kleiIMmjViJINWk80sOQw5lEzse1ZmvD+S/s8p8CWapftSaC+kocGTx6xrbrJ4WmYQGC08ffHLr6ToR6Q==", - "dev": true, "license": "MIT", "dependencies": { "@ioredis/commands": "1.4.0", @@ -8773,6 +9552,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-generator-function": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.0.tgz", @@ -9476,18 +10265,23 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true, + "license": "MIT" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", - "dev": true, "license": "MIT" }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", - "dev": true, "license": "MIT" }, "node_modules/lodash.merge": { @@ -9497,11 +10291,17 @@ "dev": true, "license": "MIT" }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/luxon": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -9633,6 +10433,13 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/module-details-from-path": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.4.tgz", + "integrity": "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==", + "dev": true, + "license": "MIT" + }, "node_modules/mri": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mri/-/mri-1.2.0.tgz", @@ -9657,14 +10464,12 @@ "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, "license": "MIT" }, "node_modules/msgpackr": { "version": "1.11.5", "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz", "integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==", - "dev": true, "license": "MIT", "optionalDependencies": { "msgpackr-extract": "^3.0.2" @@ -9674,7 +10479,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz", "integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -9723,7 +10527,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", "integrity": "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==", - "dev": true, "license": "MIT" }, "node_modules/node-fetch-native": { @@ -9736,7 +10539,6 @@ "version": "5.2.2", "resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz", "integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -10352,6 +11154,31 @@ "license": "MIT", "optional": true }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -10426,7 +11253,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -10446,7 +11272,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", - "dev": true, "license": "MIT", "dependencies": { "redis-errors": "^1.0.0" @@ -10499,6 +11324,30 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-in-the-middle": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-8.0.1.tgz", + "integrity": "sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.3.5", + "module-details-from-path": "^1.0.3" + }, + "engines": { + "node": ">=9.3.0 || >=8.10.0 <9.0.0" + } + }, "node_modules/resolve": { "version": "1.22.10", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", @@ -10691,7 +11540,6 @@ "version": "7.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.2.tgz", "integrity": "sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -10922,7 +11770,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", - "dev": true, "license": "MIT" }, "node_modules/stop-iteration-iterator": { @@ -10939,6 +11786,21 @@ "node": ">= 0.4" } }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.10", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz", @@ -10998,6 +11860,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -11432,7 +12307,6 @@ "version": "2.4.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==", - "dev": true, "license": "0BSD" }, "node_modules/type-check": { @@ -11625,7 +12499,6 @@ "version": "11.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", - "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -11871,6 +12744,34 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yallist": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", @@ -11887,8 +12788,6 @@ "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", "dev": true, "license": "ISC", - "optional": true, - "peer": true, "bin": { "yaml": "bin.mjs" }, @@ -11896,6 +12795,35 @@ "node": ">= 14.6" } }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index b460c56..47ae828 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,16 @@ "@eslint/js": "^9.18.0", "@hono/node-server": "^1.19.5", "@iconify/svelte": "^5.0.2", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-logs-otlp-grpc": "^0.211.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.211.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.211.0", + "@opentelemetry/instrumentation": "^0.211.0", + "@opentelemetry/resources": "^2.5.0", + "@opentelemetry/sdk-logs": "^0.211.0", + "@opentelemetry/sdk-metrics": "^2.5.0", + "@opentelemetry/sdk-node": "^0.211.0", + "@opentelemetry/semantic-conventions": "^1.39.0", "@sveltejs/adapter-node": "^5.2.12", "@sveltejs/kit": "^2.22.0", "@sveltejs/vite-plugin-svelte": "^6.0.0", @@ -58,6 +68,7 @@ "@aws-sdk/client-s3": "^3.907.0", "@aws-sdk/client-sts": "^3.907.0", "@prisma/client": "^6.15.0", + "bullmq-otel": "^1.1.1", "prisma": "^6.15.0", "s3-sync-client": "^4.3.1" }, diff --git a/src/hooks.server.ts b/src/hooks.server.ts index 1f80fc2..809e439 100644 --- a/src/hooks.server.ts +++ b/src/hooks.server.ts @@ -1,6 +1,8 @@ -import { type Handle, error } from '@sveltejs/kit'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; +import { type Handle, type HandleServerError, error } from '@sveltejs/kit'; import { sequence } from '@sveltejs/kit/hooks'; import { building } from '$app/environment'; +import OTEL from '$lib/otel'; import { tryVerifyAPIToken, tryVerifyCookie } from '$lib/server/auth'; import { QueueConnected, getQueues } from '$lib/server/bullmq'; import { bullboardHandle } from '$lib/server/bullmq/BullBoard'; @@ -25,6 +27,8 @@ const handleAuthRoute: Handle = async ({ event, resolve }) => { }; if (!building) { + // Start OTEL collector + OTEL.instance.start(); // Otherwise valkey will never connect and the server will always 503 getQueues(); // Likewise, initialize the Prisma connection heartbeat @@ -32,6 +36,7 @@ if (!building) { // Graceful shutdown process.on('sveltekit:shutdown', async () => { + OTEL.instance.logger.info('Shutting down gracefully...'); await Promise.all( allWorkers.map((worker) => { worker.worker?.close(); @@ -56,17 +61,102 @@ const heartbeat: Handle = async ({ event, resolve }) => { return resolve(event); }; +const tracer = trace.getTracer('IncomingRequest'); + +const authSequence: Handle = async ({ event, resolve }) => + event.route.id?.split('/')?.[1] === '(api)' + ? handleAPIRoute({ event, resolve }) + : handleAuthRoute({ event, resolve }); + export const handle: Handle = async ({ event, resolve }) => { if (event.url.pathname.startsWith('/.well-known/appspecific/')) { // Ignore these requests without logging them` return new Response('', { status: 404 }); } - return await sequence( - heartbeat, - (h) => { - return event.route.id?.split('/')?.[1] === '(api)' ? handleAPIRoute(h) : handleAuthRoute(h); - }, - bullboardHandle - )({ event, resolve }); + return tracer.startActiveSpan(`${event.request.method} ${event.url.pathname}`, async (span) => { + let clientIp; + try { + clientIp = event.getClientAddress(); + } catch (e) { + span.recordException(e as Error); + clientIp = 'unknown'; + } + span.setAttributes({ + 'http.method': event.request.method, + 'http.url': event.url.href, + 'http.route': event.url.pathname ?? '', + 'http.user_agent': event.request.headers.get('user-agent') ?? '', + 'http.client_ip': clientIp, + 'http.x-forwarded-for': event.request.headers.get('x-forwarded-for') ?? '', + 'svelte.route_id': event.route.id ?? '' + }); + try { + const response = await sequence( + heartbeat, + // Handle auth hooks in a separate OTEL span + (h) => { + return tracer.startActiveSpan('Authentication', async (span) => { + // Call the auth sequence + let spanEnded = false; + try { + const ret = await authSequence({ + event, + resolve: (...args) => { + if (!spanEnded) { + span.end(); + spanEnded = true; + } + return resolve(...args); + } + }); + return ret; + } finally { + if (!spanEnded) { + span.end(); + spanEnded = true; + } + } + }); + }, + bullboardHandle + )({ event, resolve }); + span.setAttributes({ + 'http.status_code': response.status + }); + return response; + } finally { + span.end(); + } + }); +}; + +export const handleError: HandleServerError = ({ error, event, status }) => { + // Log the error with OTEL + OTEL.instance.logger.error('Error in handleError', { + error: error instanceof Error ? error.message : String(error), + route: event.route.id, + method: event.request.method, + url: event.url.href + }); + trace.getActiveSpan()?.recordException(error as Error); + trace.getActiveSpan()?.setStatus({ + code: SpanStatusCode.ERROR, // Error + message: error instanceof Error ? error.message : String(error) + }); + + if (status === 404) { + // Don't log 404 errors, they are common and not actionable + return { + message: 'Not found', + status: 404 + }; + } + + console.error('Error occurred:', error); + + return { + message: 'An unexpected error occurred. Please try again later.', + status: 500 // Internal Server Error + }; }; diff --git a/src/lib/otel/development_config.yml b/src/lib/otel/development_config.yml new file mode 100644 index 0000000..24e46b2 --- /dev/null +++ b/src/lib/otel/development_config.yml @@ -0,0 +1,21 @@ +processors: + batch: +exporters: + debug: + # verbosity: detailed + otlp: + endpoint: jaeger:4317 + tls: + insecure: true +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [debug,otlp] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/src/lib/otel/index.ts b/src/lib/otel/index.ts new file mode 100644 index 0000000..6d75e27 --- /dev/null +++ b/src/lib/otel/index.ts @@ -0,0 +1,114 @@ +// This file should ideally be imported first in the application to ensure +// that OpenTelemetry is initialized before any other modules are loaded. + +// Adding OTEL instrumentation hooks breaks the build +// It should look like this but breaks imports later + +// import { register } from 'node:module'; +// // eslint-disable-next-line import/order +// import { pathToFileURL } from 'node:url'; +// register('@opentelemetry/instrumentation/hook.mjs', pathToFileURL('./')); + +import { context } from '@opentelemetry/api'; +import type { AnyValueMap, Logger as OTLogger } from '@opentelemetry/api-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs'; +import { PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; + +class Logger { + constructor( + private logger: OTLogger, + private enableDev: boolean + ) {} + info(message: string, attributes: AnyValueMap = {}) { + this.logger.emit({ + body: message, + severityText: 'INFO', + attributes, + context: context.active() + }); + } + dev(message: string, attributes: AnyValueMap = {}) { + if (this.enableDev) { + this.logger.emit({ + body: message, + severityText: 'INFO - DEV', + attributes, + context: context.active() + }); + } + } + error(message: string, attributes: AnyValueMap = {}) { + this.logger.emit({ + body: message, + severityText: 'ERROR', + attributes, + context: context.active() + }); + } +} + +export default class OTEL { + private static _instance: OTEL; + private sdk: NodeSDK; + private initialized = false; + private _logger: Logger; + + private constructor() { + const isDev = process.env.NODE_ENV === 'development'; + const endpoint = `http://${isDev ? 'localhost' : 'otel'}:6317`; + + const resource = resourceFromAttributes({ + [ATTR_SERVICE_NAME]: process.env.APP_ENV + '_build-engine', + [ATTR_SERVICE_VERSION]: '1.0' + }); + + const logProcessor = new SimpleLogRecordProcessor( + new OTLPLogExporter({ + url: endpoint + }) + ); + + this._logger = new Logger( + new LoggerProvider({ + resource, + processors: [logProcessor] + }).getLogger(process.env.APP_ENV + '_build-engine-logger'), + isDev + ); + + this.sdk = new NodeSDK({ + resource, + traceExporter: new OTLPTraceExporter({ url: endpoint }), + metricReaders: [ + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ url: endpoint }) + }) + ], + logRecordProcessors: [logProcessor] + }); + } + + public static get instance(): OTEL { + if (!OTEL._instance) { + OTEL._instance = new OTEL(); + } + return OTEL._instance; + } + + public start() { + if (!this.initialized) { + this.initialized = true; + this.sdk.start(); + } + } + + public get logger() { + return this._logger; + } +} diff --git a/src/lib/otel/production_config.yml b/src/lib/otel/production_config.yml new file mode 100644 index 0000000..21bd4c9 --- /dev/null +++ b/src/lib/otel/production_config.yml @@ -0,0 +1,123 @@ +processors: + memory_limiter: + check_interval: 5s + limit_mib: 500 + spike_limit_mib: 100 + batch: + # Add SampleRate attribute to traces for Honeycomb automatic accounting + # SampleRate = N where we keep 1/N traces, or (sampled traces)/(real traces) + # Higher N means lower sampling rate + # N = 4 means 1/4 = 25% of traces sampled + # N = 2 means 1/2 = 50% of traces sampled + # N = 1.25 means 1/1.25 = 80% of traces sampled + # N = 1 means 100% of traces sampled + attributes/noisy: + actions: + - key: SampleRate + action: insert + value: 2 + attributes/default: + actions: + - key: SampleRate + action: insert + value: 1.25 + attributes/forced: + actions: + - key: SampleRate + action: insert + value: 1 + tail_sampling: + decision_wait: 10s + num_traces: 1000 + policies: + # Keep 5% of /admin/jobs requests (noisy route) + - name: Throttle noisy routes + type: and + and: + and_sub_policy: + - name: Match noisy routes + type: string_attribute + string_attribute: + key: svelte.route_id + values: ["/(ui)/queue-admin/[...rest]"] + - name: Throttle noisy routes + type: probabilistic + probabilistic: + sampling_percentage: 5.0 + # Sample 80% of all traces + - name: Sample 80% of all traces + type: probabilistic + probabilistic: + sampling_percentage: 80.0 + # Keep all error traces + - name: Keep all error traces + type: status_code + status_code: { status_codes: [ERROR] } + # Keep traces with force_sample + - name: Keep traces with force_sample + type: boolean_attribute + boolean_attribute: + key: force_sample + value: true + # Drop traces with do_not_sample + - name: Drop traces with do_not_sample + type: drop + drop: + drop_sub_policy: + - type: boolean_attribute + boolean_attribute: + key: do_not_sample + value: true + +connectors: + routing: + default_pipelines: [traces/default] + table: + - context: span + condition: resource.attributes["svelte.route_id"] == "/(ui)/queue-admin/[...rest]" + pipelines: [traces/noisy] + - context: span + condition: resource.attributes["force_sample"] == true or span.status.code == STATUS_CODE_ERROR + pipelines: [traces/forced] + +exporters: + debug: + otlp: + endpoint: api.honeycomb.io:443 + headers: + 'x-honeycomb-team': ${env:HONEYCOMB_API_KEY} + otlp/metrics: + endpoint: api.honeycomb.io:443 + headers: + 'x-honeycomb-team': ${env:HONEYCOMB_API_KEY} + 'x-honeycomb-dataset': 'appbuilder-buildengine-api-metrics' + otlp/logs: + endpoint: api.honeycomb.io:443 + headers: + 'x-honeycomb-team': ${env:HONEYCOMB_API_KEY} + 'x-honeycomb-dataset': 'appbuilder-buildengine-api-logs' + +service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, tail_sampling] + exporters: [routing] + traces/noisy: + receivers: [routing] + processors: [attributes/noisy, batch] + exporters: [debug, otlp] + traces/default: + receivers: [routing] + processors: [attributes/default, batch] + exporters: [debug, otlp] + traces/forced: + receivers: [routing] + processors: [attributes/forced, batch] + exporters: [debug, otlp] + metrics: + receivers: [otlp] + exporters: [debug, otlp/metrics] + logs: + receivers: [otlp] + exporters: [debug, otlp/logs] diff --git a/src/lib/otel/receivers_config.yml b/src/lib/otel/receivers_config.yml new file mode 100644 index 0000000..d6cfe16 --- /dev/null +++ b/src/lib/otel/receivers_config.yml @@ -0,0 +1,7 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:6317 + http: + endpoint: 0.0.0.0:6318 diff --git a/src/lib/server/auth.ts b/src/lib/server/auth.ts index 84d8e37..cdfa4c2 100644 --- a/src/lib/server/auth.ts +++ b/src/lib/server/auth.ts @@ -1,3 +1,4 @@ +import { trace } from '@opentelemetry/api'; import type { Prisma } from '@prisma/client'; import { type RequestEvent, error, redirect } from '@sveltejs/kit'; import { jwtDecrypt } from 'jose'; @@ -17,6 +18,8 @@ export async function tryVerifyCookie(event: RequestEvent, gotoLoginPage = true) token = await jwtDecrypt(cookie, new TextEncoder().encode(secrets.AUTH0_SECRET)); event.locals.userEmail = token.payload.email as string; + + trace.getActiveSpan()?.setAttribute('user.email', event.locals.userEmail); } } catch { /* empty */ @@ -41,6 +44,11 @@ async function initiateScriptoriaLogin(event: RequestEvent) { const verify = randomUUID(); const requestId = randomUUID(); + trace.getActiveSpan()?.setAttributes({ + 'scriptoria-endpoint': env.PUBLIC_SCRIPTORIA_URL, + 'request-id': requestId + }); + await getAuthConnection().set(`${requestId}`, verify, 'EX', 300); // 5 minute (300 s) TTL const hash = createHash('sha256'); @@ -95,5 +103,7 @@ export async function tryVerifyAPIToken( return [false, ErrorResponse(403, 'Invalid Access Token')]; } + trace.getActiveSpan()?.setAttributes({ 'client.id': client.id, 'client.prefix': client.prefix }); + return [true, client]; } diff --git a/src/lib/server/aws/codebuild.ts b/src/lib/server/aws/codebuild.ts index 7327889..a8693da 100644 --- a/src/lib/server/aws/codebuild.ts +++ b/src/lib/server/aws/codebuild.ts @@ -8,6 +8,7 @@ import { type ProjectSource, StartBuildCommand } from '@aws-sdk/client-codebuild'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; import type { Prisma } from '@prisma/client'; import { AWSCommon } from './common'; import { S3 } from './s3'; @@ -18,7 +19,6 @@ import { getBasePrefixUrl } from '$lib/server/models/artifacts'; import { Job } from '$lib/server/models/job'; -import { Utils } from '$lib/server/utils'; export type ReleaseForCodeBuild = Prisma.releaseGetPayload<{ select: { @@ -45,6 +45,8 @@ export type BuildForCodeBuild = Prisma.buildGetPayload<{ }; }>; +const tracer = trace.getTracer('CodeBuild'); + export class CodeBuild extends AWSCommon { public codeBuildClient; public constructor() { @@ -78,39 +80,80 @@ export class CodeBuild extends AWSCommon { versionCode: number, codeCommit: boolean ) { - const prefix = Utils.getPrefix(); - const job = build.job; - const buildProcess = `build_${job.app_id}`; - const jobNumber = String(job.id); - const buildNumber = String(build.id); - console.log( - `[${prefix}] startBuild CodeBuild Project: ${buildProcess} URL: ${repoUrl} commitId: ${commitId} jobNumber: ${jobNumber} buildNumber: ${buildNumber} versionCode: ${versionCode}` - ); - const artifacts_bucket = CodeBuild.getArtifactsBucket(); - const secretsBucket = CodeBuild.getSecretsBucket(); - const buildApp = CodeBuild.getCodeBuildProjectName('build_app'); - const buildPath = this.getBuildPath(job); - const artifactPath = getArtifactPath(job, 'codebuild-output'); - console.log(`Artifacts path: ${artifactPath}`); - // Leaving all this code together to make it easier to remove when git is no longer supported - if (codeCommit) { - console.log(`[${prefix}] startBuild CodeCommit Project`); - const res = await this.codeBuildClient.send( - new StartBuildCommand({ - projectName: buildApp, - artifactsOverride: { - location: artifacts_bucket, - name: '/', - namespaceType: 'NONE', - packaging: 'NONE', - path: artifactPath, - type: 'S3' - }, - buildspecOverride: buildSpec, - environmentVariablesOverride: [ + return tracer.startActiveSpan(`CodeBuild - StartBuild`, async (span) => { + try { + const job = build.job; + const buildProcess = `build_${job.app_id}`; + span.setAttributes({ + 'code-build.process': buildProcess, + 'code-build.repo-url': repoUrl, + 'code-build.commit-id': commitId, + 'code-build.job-id': job.id, + 'code-build.build-id': build.id, + 'code-build.version-code': versionCode + }); + const artifacts_bucket = CodeBuild.getArtifactsBucket(); + const secretsBucket = CodeBuild.getSecretsBucket(); + const buildApp = CodeBuild.getCodeBuildProjectName('build_app'); + const buildPath = this.getBuildPath(job); + const artifactPath = getArtifactPath(job, 'codebuild-output'); + span.setAttribute('code-build.artifact-path', artifactPath); + // Leaving all this code together to make it easier to remove when git is no longer supported + if (codeCommit) { + span.addEvent('StartBuild - CodeCommit'); + const res = await this.codeBuildClient.send( + new StartBuildCommand({ + projectName: buildApp, + artifactsOverride: { + location: artifacts_bucket, + name: '/', + namespaceType: 'NONE', + packaging: 'NONE', + path: artifactPath, + type: 'S3' + }, + buildspecOverride: buildSpec, + environmentVariablesOverride: [ + { + name: 'BUILD_NUMBER', + value: String(build.id) + }, + { + name: 'APP_BUILDER_SCRIPT_PATH', + value: buildPath + }, + { + name: 'PUBLISHER', + value: job.publisher_id + }, + { + name: 'VERSION_CODE', + value: '' + versionCode + }, + { + name: 'SECRETS_BUCKET', + value: secretsBucket + } + ], + sourceLocationOverride: repoUrl, + sourceVersion: commitId + }) + ); + const buildId = res.build?.id; + const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); + span.setAttributes({ + 'code-build.build.build-id': buildId, + 'code-build.build.build-guid': buildGuid + }); + return buildGuid; + } else { + span.addEvent('StartBuild - S3'); + const targets = build.targets ?? 'apk play-listing'; + const environmentArray = [ + // BUILD_NUMBER Must be first for tests { name: 'BUILD_NUMBER', - value: buildNumber + value: String(build.id) }, { name: 'APP_BUILDER_SCRIPT_PATH', @@ -127,83 +170,62 @@ export class CodeBuild extends AWSCommon { { name: 'SECRETS_BUCKET', value: secretsBucket + }, + { + name: 'PROJECT_S3', + value: repoUrl + }, + { + name: 'TARGETS', + value: targets + }, + { + name: 'SCRIPT_S3', + value: S3.getBuildScriptPath() } - ], - sourceLocationOverride: repoUrl, - sourceVersion: commitId - }) - ); - const buildId = res.build?.id; - const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); - console.log(`Build id: ${buildId} Guid: ${buildGuid}`); - return buildGuid; - } else { - console.log(`[${prefix}] startBuild S3 Project`); - const targets = build.targets ?? 'apk play-listing'; - const environmentArray = [ - // BUILD_NUMBER Must be first for tests - { - name: 'BUILD_NUMBER', - value: buildNumber - }, - { - name: 'APP_BUILDER_SCRIPT_PATH', - value: buildPath - }, - { - name: 'PUBLISHER', - value: job.publisher_id - }, - { - name: 'VERSION_CODE', - value: '' + versionCode - }, - { - name: 'SECRETS_BUCKET', - value: secretsBucket - }, - { - name: 'PROJECT_S3', - value: repoUrl - }, - { - name: 'TARGETS', - value: targets - }, - { - name: 'SCRIPT_S3', - value: S3.getBuildScriptPath() + ]; + const adjustedEnvironmentArray = this.addEnvironmentToArray( + environmentArray, + build.environment + ); + const computeType = this.getComputeType(adjustedEnvironmentArray); + const imageTag = this.getImageTag(adjustedEnvironmentArray); + const result = await this.codeBuildClient.send( + new StartBuildCommand({ + projectName: buildApp, + artifactsOverride: { + location: artifacts_bucket, // output bucket + name: '/', // name of output artifact object + namespaceType: 'NONE', + packaging: 'NONE', + path: artifactPath, // path to output artifacts + type: 'S3' // REQUIRED + }, + buildspecOverride: buildSpec, + environmentVariablesOverride: adjustedEnvironmentArray, + sourceTypeOverride: 'NO_SOURCE', + computeTypeOverride: computeType, + imageOverride: CodeBuild.getCodeBuildImageRepo() + ':' + imageTag + }) + ); + const buildId = result.build?.id; + const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); + span.setAttributes({ + 'code-build.build.build-id': buildId, + 'code-build.build.build-guid': buildGuid + }); + return buildGuid; } - ]; - const adjustedEnvironmentArray = this.addEnvironmentToArray( - environmentArray, - build.environment - ); - const computeType = this.getComputeType(adjustedEnvironmentArray); - const imageTag = this.getImageTag(adjustedEnvironmentArray); - const result = await this.codeBuildClient.send( - new StartBuildCommand({ - projectName: buildApp, - artifactsOverride: { - location: artifacts_bucket, // output bucket - name: '/', // name of output artifact object - namespaceType: 'NONE', - packaging: 'NONE', - path: artifactPath, // path to output artifacts - type: 'S3' // REQUIRED - }, - buildspecOverride: buildSpec, - environmentVariablesOverride: adjustedEnvironmentArray, - sourceTypeOverride: 'NO_SOURCE', - computeTypeOverride: computeType, - imageOverride: CodeBuild.getCodeBuildImageRepo() + ':' + imageTag - }) - ); - const buildId = result.build?.id; - const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); - console.log(`Build id: ${buildId} Guid: ${buildGuid}`); - return buildGuid; - } + } catch (e) { + span.recordException(e as Error); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span.end(); + } + }); } /** @@ -214,16 +236,27 @@ export class CodeBuild extends AWSCommon { * @return AWS/Result Result object on the status of the build */ public async getBuildStatus(guid: string, buildProcess: string) { - const prefix = Utils.getPrefix(); - console.log(`[${prefix}] getBuildStatus CodeBuild Project: ${buildProcess} BuildGuid: ${guid}`); + const span = trace.getActiveSpan(); + try { + span?.setAttributes({ + 'code-build.build-process': buildProcess, + 'code-build.build-guid': guid + }); - const buildId = this.getBuildId(guid, buildProcess); - const result = await this.codeBuildClient.send( - new BatchGetBuildsCommand({ - ids: [buildId] - }) - ); - return result.builds?.at(0); + const buildId = this.getBuildId(guid, buildProcess); + const result = await this.codeBuildClient.send( + new BatchGetBuildsCommand({ + ids: [buildId] + }) + ); + return result.builds?.at(0); + } catch (e) { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } } /** @@ -249,9 +282,7 @@ export class CodeBuild extends AWSCommon { * @return string CodeBuild build arn */ private getBuildId(guid: string, buildProcess: string) { - const buildId = `${buildProcess}:${guid}`; - console.log(`getBuildId arn: ${buildId}`); - return buildId; + return `${buildProcess}:${guid}`; } /** * Returns the name of the shell command to be run @@ -281,99 +312,119 @@ export class CodeBuild extends AWSCommon { * @return false|string */ public async startRelease(release: ReleaseForCodeBuild, releaseSpec: string) { - console.log('startRelease: '); - const releaseNumber = '' + release.id; - const build = release.build; - const buildNumber = '' + build.id; - const job = build.job; - const buildPath = this.getBuildPath(job); - const artifacts_bucket = CodeBuild.getArtifactsBucket(); - const artifactPath = getArtifactPath(job, 'codebuild-output', true); - const secretsBucket = CodeBuild.getSecretsBucket(); - const scriptureEarthKey = CodeBuild.getScriptureEarthKey(); - const publishApp = CodeBuild.getCodeBuildProjectName('publish_app'); - const promoteFrom = release.promote_from ?? ''; + return tracer.startActiveSpan(`CodeBuild - StartRelease`, async (span) => { + try { + const build = release.build; + const job = build.job; + const buildPath = this.getBuildPath(job); + const artifacts_bucket = CodeBuild.getArtifactsBucket(); + const artifactPath = getArtifactPath(job, 'codebuild-output', true); + const secretsBucket = CodeBuild.getSecretsBucket(); + const scriptureEarthKey = CodeBuild.getScriptureEarthKey(); + const publishApp = CodeBuild.getCodeBuildProjectName('publish_app'); + const promoteFrom = release.promote_from ?? ''; - const sourceLocation = this.getSourceLocation(build); - const s3Artifacts = this.getArtifactsLocation(build); - console.log(`Source location: ${sourceLocation}`); - const targets = release.targets ?? 'google-play'; + const sourceLocation = this.getSourceLocation(build); + const s3Artifacts = this.getArtifactsLocation(build); + const targets = release.targets ?? 'google-play'; - const environmentArray = [ - // RELEASE_NUMBER Must be first for tests - { - name: 'RELEASE_NUMBER', - value: releaseNumber - }, - { - name: 'APP_BUILDER_SCRIPT_PATH', - value: buildPath - }, - { - name: 'BUILD_NUMBER', - value: buildNumber - }, - { - name: 'CHANNEL', - value: release.channel - }, - { - name: 'PUBLISHER', - value: job.publisher_id - }, - { - name: 'PROJECT_S3', - value: job.git_url - }, - { - name: 'SECRETS_BUCKET', - value: secretsBucket - }, - { - name: 'PROMOTE_FROM', - value: promoteFrom - }, - { - name: 'ARTIFACTS_S3_DIR', - value: s3Artifacts - }, - { - name: 'TARGETS', - value: targets - }, - { - name: 'SCRIPT_S3', - value: S3.getBuildScriptPath() - }, - { - name: 'SCRIPTURE_EARTH_KEY', - value: scriptureEarthKey + span.setAttributes({ + 'code-build.release-id': release.id, + 'code-build.build-id': build.id, + 'code-build.artifact-path': artifactPath, + 'code-build.promote-from': promoteFrom, + 'code-build.source': sourceLocation, + 'code-build.targets': targets + }); + + const environmentArray = [ + // RELEASE_NUMBER Must be first for tests + { + name: 'RELEASE_NUMBER', + value: String(release.id) + }, + { + name: 'APP_BUILDER_SCRIPT_PATH', + value: buildPath + }, + { + name: 'BUILD_NUMBER', + value: String(build.id) + }, + { + name: 'CHANNEL', + value: release.channel + }, + { + name: 'PUBLISHER', + value: job.publisher_id + }, + { + name: 'PROJECT_S3', + value: job.git_url + }, + { + name: 'SECRETS_BUCKET', + value: secretsBucket + }, + { + name: 'PROMOTE_FROM', + value: promoteFrom + }, + { + name: 'ARTIFACTS_S3_DIR', + value: s3Artifacts + }, + { + name: 'TARGETS', + value: targets + }, + { + name: 'SCRIPT_S3', + value: S3.getBuildScriptPath() + }, + { + name: 'SCRIPTURE_EARTH_KEY', + value: scriptureEarthKey + } + ]; + const adjustedEnvironmentArray = this.addEnvironmentToArray( + environmentArray, + release.environment + ); + const result = await this.codeBuildClient.send( + new StartBuildCommand({ + projectName: publishApp, + artifactsOverride: { + location: artifacts_bucket, // output bucket + name: '/', // name of output artifact object + namespaceType: 'NONE', + packaging: 'NONE', + path: artifactPath, // path to output artifacts + type: 'S3' // REQUIRED + }, + buildspecOverride: releaseSpec, + environmentVariablesOverride: adjustedEnvironmentArray, + sourceLocationOverride: sourceLocation + }) + ); + const buildId = result.build?.id; + const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); + span.setAttributes({ + 'code-build.release.build-id': buildId, + 'code-build.release.build-guid': buildGuid + }); + return buildGuid; + } catch (e) { + span.recordException(e as Error); + span.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span.end(); } - ]; - const adjustedEnvironmentArray = this.addEnvironmentToArray( - environmentArray, - release.environment - ); - const result = await this.codeBuildClient.send( - new StartBuildCommand({ - projectName: publishApp, - artifactsOverride: { - location: artifacts_bucket, // output bucket - name: '/', // name of output artifact object - namespaceType: 'NONE', - packaging: 'NONE', - path: artifactPath, // path to output artifacts - type: 'S3' // REQUIRED - }, - buildspecOverride: releaseSpec, - environmentVariablesOverride: adjustedEnvironmentArray, - sourceLocationOverride: sourceLocation - }) - ); - const buildId = result.build?.id; - const buildGuid = buildId?.substring(buildId.indexOf(':') + 1); - console.log(`Build id: ${buildId} Guid: ${buildGuid}`); - return buildGuid; + }); } /** * Get the url for the apk file in a format that codebuild accepts for an S3 Source @@ -422,7 +473,6 @@ export class CodeBuild extends AWSCommon { ) { const project_name = CodeBuild.getCodeBuildProjectName(base_name); const artifacts_bucket = CodeBuild.getArtifactsBucket(); - console.log(`Bucket: ${artifacts_bucket}`); return await this.codeBuildClient.send( new CreateProjectCommand({ artifacts: { @@ -470,14 +520,24 @@ export class CodeBuild extends AWSCommon { * @return boolean true if project found */ public async projectExists(baseName: string) { - const projectName = CodeBuild.getCodeBuildProjectName(baseName); - console.log(`Check project ${projectName} exists`); - const result = await this.codeBuildClient.send( - new BatchGetProjectsCommand({ - names: [projectName] - }) - ); - return !!result.projects?.length; + let exists = false; + try { + const projectName = CodeBuild.getCodeBuildProjectName(baseName); + trace?.getActiveSpan()?.setAttribute('code-build.project-name', projectName); + const result = await this.codeBuildClient.send( + new BatchGetProjectsCommand({ + names: [projectName] + }) + ); + exists = !!result.projects?.length; + } catch (e) { + trace.getActiveSpan()?.recordException(e as Error); + trace.getActiveSpan()?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } + return exists; } private addEnvironmentToArray( @@ -498,7 +558,7 @@ export class CodeBuild extends AWSCommon { ) ); } catch { - console.log('Exception caught and ignored'); + /* empty */ } } return environmentVariables; diff --git a/src/lib/server/aws/codecommit.ts b/src/lib/server/aws/codecommit.ts index 0305686..1d44a64 100644 --- a/src/lib/server/aws/codecommit.ts +++ b/src/lib/server/aws/codecommit.ts @@ -3,8 +3,8 @@ import { GetBranchCommand, GetRepositoryCommand } from '@aws-sdk/client-codecommit'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; import { AWSCommon } from './common'; -import { Utils } from '$lib/server/utils'; export class CodeCommit extends AWSCommon { public codeCommitClient; @@ -18,9 +18,25 @@ export class CodeCommit extends AWSCommon { * @return \Aws\CodeBuild\CodeCommitClient */ public static getCodeCommitClient() { - return new CodeCommitClient({ - region: AWSCommon.getArtifactsBucketRegion() - }); + const span = trace.getActiveSpan(); + let client: CodeCommitClient | null = null; + try { + client = new CodeCommitClient({ + region: AWSCommon.getArtifactsBucketRegion() + }); + } catch (e) { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span?.addEvent('CodeCommit - getCodeCommitClient', { + 'code-commit.client.api-version': client?.config.apiVersion, + 'code-commit.client.service-id': client?.config.serviceId + }); + } + return client!; } /** * Returns http url of code commit archive derived from git url needed for CodeBuild @@ -29,15 +45,28 @@ export class CodeCommit extends AWSCommon { * @return string http codecommit url */ public async getSourceURL(git_url: string) { - console.log(`[${Utils.getPrefix()}] getSourceURL URL: ${git_url}`); - const repo = git_url.substring(git_url.indexOf('/') + 1); - const repoInfo = await this.codeCommitClient.send( - new GetRepositoryCommand({ - repositoryName: repo - }) - ); - const cloneUrl = repoInfo.repositoryMetadata?.cloneUrlHttp; - console.log(`cloneUrl: ${cloneUrl}`); + const span = trace.getActiveSpan(); + let cloneUrl: string | undefined = undefined; + try { + const repo = git_url.substring(git_url.indexOf('/') + 1); + const repoInfo = await this.codeCommitClient.send( + new GetRepositoryCommand({ + repositoryName: repo + }) + ); + cloneUrl = repoInfo.repositoryMetadata?.cloneUrlHttp; + } catch (e) { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span?.addEvent('CodeCommit - getSourceURL', { + 'code-commit.git-url': git_url, + 'code-commit.source-url': cloneUrl ?? '' + }); + } return cloneUrl; } /** @@ -47,15 +76,28 @@ export class CodeCommit extends AWSCommon { * @return string http codecommit url */ public async getSourceSshURL(git_url: string) { - console.log(`[${Utils.getPrefix()}] getSourceURL URL: ${git_url}`); - const repo = git_url.substring(git_url.indexOf('/') + 1); - const repoInfo = await this.codeCommitClient.send( - new GetRepositoryCommand({ - repositoryName: repo - }) - ); - const cloneUrl = repoInfo.repositoryMetadata?.cloneUrlSsh; - console.log(`cloneUrl: ${cloneUrl}`); + const span = trace.getActiveSpan(); + let cloneUrl: string | undefined = undefined; + try { + const repo = git_url.substring(git_url.indexOf('/') + 1); + const repoInfo = await this.codeCommitClient.send( + new GetRepositoryCommand({ + repositoryName: repo + }) + ); + cloneUrl = repoInfo.repositoryMetadata?.cloneUrlSsh; + } catch (e) { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span?.addEvent('CodeCommit - getSourceSshURL', { + 'code-commit.git-url': git_url, + 'code-commit.source-ssh-url': cloneUrl ?? '' + }); + } return cloneUrl; } @@ -67,16 +109,29 @@ export class CodeCommit extends AWSCommon { * @return string commit id */ public async getCommitId(git_url: string, branch: string) { - console.log(`[${Utils.getPrefix()}] getCommitId URL: ${git_url} Branch: ${branch}`); - const repo = git_url.substring(git_url.indexOf('/') + 1); - const result = await this.codeCommitClient.send( - new GetBranchCommand({ - branchName: branch, - repositoryName: repo - }) - ); - const commitId = result.branch?.commitId; - console.log(`commitId: ${commitId}`); + const span = trace.getActiveSpan(); + let commitId: string | undefined = undefined; + try { + const repo = git_url.substring(git_url.indexOf('/') + 1); + const result = await this.codeCommitClient.send( + new GetBranchCommand({ + branchName: branch, + repositoryName: repo + }) + ); + commitId = result.branch?.commitId; + } catch (e) { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + } finally { + span?.addEvent('CodeCommit - getCommitId', { + 'code-commit.git-url': git_url, + 'code-commit.commit-id': commitId ?? '' + }); + } return commitId; } } diff --git a/src/lib/server/aws/iamwrapper.ts b/src/lib/server/aws/iamwrapper.ts index c90c02b..8615dde 100644 --- a/src/lib/server/aws/iamwrapper.ts +++ b/src/lib/server/aws/iamwrapper.ts @@ -12,6 +12,7 @@ import { RemoveUserFromGroupCommand, UploadSSHPublicKeyCommand } from '@aws-sdk/client-iam'; +import { trace } from '@opentelemetry/api'; import { AWSCommon } from './common'; import { env } from '$env/dynamic/private'; @@ -54,7 +55,6 @@ export class IAmWrapper extends AWSCommon { public async doesRoleExist(projectName: string) { try { const fullRoleName = AWSCommon.getRoleName(projectName); - console.log(`Check role ${fullRoleName} exists`); await this.iamClient.send( new GetRoleCommand({ RoleName: fullRoleName // REQUIRED @@ -79,7 +79,7 @@ export class IAmWrapper extends AWSCommon { }) ); const roleArn = result.Role?.Arn ?? ''; - console.log(`Role Arn is ${roleArn}`); + trace.getActiveSpan()?.setAttribute('iam.role-arn', roleArn); return roleArn; } catch { return ''; diff --git a/src/lib/server/aws/s3.ts b/src/lib/server/aws/s3.ts index 8343c16..6a33637 100644 --- a/src/lib/server/aws/s3.ts +++ b/src/lib/server/aws/s3.ts @@ -9,6 +9,7 @@ import { S3ServiceException, type _Object } from '@aws-sdk/client-s3'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; import { basename, dirname, extname } from 'node:path'; import { S3SyncClient } from 's3-sync-client'; import { AWSCommon } from './common'; @@ -21,7 +22,6 @@ import { getBasePrefixUrl, handleArtifact } from '$lib/server/models/artifacts'; -import { Utils } from '$lib/server/utils'; export class S3 extends AWSCommon { public s3Client; @@ -80,17 +80,25 @@ export class S3 extends AWSCommon { ); fileContents = await result.Body!.transformToString(); } catch (e) { + const span = trace.getActiveSpan(); + span?.recordException(e as Error); // There is not a good way to check for file exists. If file doesn't exist, // it will be caught here and an empty string returned. if (e instanceof NoSuchKey) { - console.error( - `Error from S3 while getting object "${filePath}" from "${bucket}". No such key exists.` - ); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: `Error from S3 while getting object "${filePath}" from "${bucket}". No such key exists.` + }); } else if (e instanceof S3ServiceException) { - console.error( - `Error from S3 while getting object from ${bucket}. ${e.name}: ${e.message}` - ); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: `Error from S3 while getting object from ${bucket}. ${e.name}: ${e.message}` + }); } else { + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); throw e; } } @@ -106,18 +114,26 @@ export class S3 extends AWSCommon { * @param Build or Release artifacts_provider - The build or release */ public async copyS3Folder(artifacts_provider: ProviderForPrefix & ProviderForArtifacts) { + const span = trace.getActiveSpan(); const artifactsBucket = S3.getArtifactsBucket(); const destFolderPrefix = getBasePrefixUrl(artifacts_provider, S3.getAppEnv()); const sourcePrefix = getBasePrefixUrl(artifacts_provider, 'codebuild-output') + '/'; const destPrefix = destFolderPrefix + '/'; beginArtifacts(artifacts_provider, `https://${artifactsBucket}.s3.amazonaws.com/${destPrefix}`); + span?.addEvent(`S3 - Copy Folder`, { + 's3.bucket': artifactsBucket, + 's3.source': sourcePrefix, + 's3.dest': destPrefix + }); try { await this.deleteMatchingObjects(artifactsBucket, destFolderPrefix); } catch (e) { - if (e instanceof S3ServiceException) { - console.error(`[${[Utils.getPrefix()]}] copyS3Build - Folder: Exception:\n${e}`); - } else { + span?.recordException(e as Error); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); + if (!(e instanceof S3ServiceException)) { throw e; } } @@ -147,22 +163,24 @@ export class S3 extends AWSCommon { destPrefix: string, artifacts_provider: ProviderForPrefix & ProviderForArtifacts ) { + const span = trace.getActiveSpan(); const artifactsBucket = S3.getArtifactsBucket(); let fileContents = ''; const fileNameWithPrefix = file['Key']!; const fileName = fileNameWithPrefix.substring(sourcePrefix.length); + span?.addEvent(`S3 - Copy File`, { + 's3.bucket': artifactsBucket, + 's3.source': sourcePrefix, + 's3.file': fileName + }); switch (fileName) { case 'manifest.txt': - return; case 'play-listing/default-language.txt': return; //case: 'version.json': FUTURE: get versionCode from version.json case 'version_code.txt': fileContents = await this.readS3File(artifacts_provider, fileName); break; - default: - console.log(fileName); - break; } const sourceDir = dirname(fileNameWithPrefix); const sourceBasename = basename(fileNameWithPrefix); @@ -186,9 +204,17 @@ export class S3 extends AWSCommon { } return ret; } catch (e) { + span?.recordException(e as Error); if (e instanceof Error) { - console.error(`File was not renamed ${sourceFile}\nexception: ${e.stack}`); + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: `File was not renamed ${sourceFile}\nexception: ${e.message}` + }); } else { + span?.setStatus({ + code: SpanStatusCode.ERROR, + message: (e as Error).message + }); throw e; } } @@ -203,6 +229,12 @@ export class S3 extends AWSCommon { const destPrefix: string = getBasePrefixUrl(artifacts_provider, S3.getAppEnv()); const fileS3Key = destPrefix + '/' + fileName; + trace.getActiveSpan()?.addEvent(`S3 - Write File`, { + 's3.bucket': fileS3Bucket, + 's3.dest': destPrefix, + 's3.file': fileName + }); + await this.s3Client.send( new PutObjectCommand({ Bucket: fileS3Bucket, @@ -222,11 +254,18 @@ export class S3 extends AWSCommon { public async removeCodeBuildFolder(artifacts_provider: ProviderForPrefix) { const s3Folder = getBasePrefixUrl(artifacts_provider, 'codebuild-output') + '/'; const s3Bucket = S3.getArtifactsBucket(); - console.log(`Deleting S3 bucket: ${s3Bucket} key: ${s3Folder}`); + trace.getActiveSpan()?.addEvent(`S3 - Remove CodeBuild Folder`, { + 's3.bucket': s3Bucket, + 's3.folder': s3Folder + }); return this.deleteMatchingObjects(s3Bucket, s3Folder); } public async uploadFolder(folderName: string, bucket: string) { + trace.getActiveSpan()?.addEvent(`S3 - Upload Folder`, { + 's3.bucket': bucket, + 's3.folder': folderName + }); const client = new S3SyncClient({ client: this.getS3ClientWithCredentials() }); return await client.sync(folderName, bucket); } @@ -251,6 +290,10 @@ export class S3 extends AWSCommon { } private async deleteMatchingObjects(Bucket: string, Prefix: string) { + trace.getActiveSpan()?.addEvent(`S3 - Delete Objects`, { + 's3.bucket': Bucket, + 's3.prefix': Prefix + }); // If destination folder already exists from some previous build, delete const existing = await this.s3Client.send( new ListObjectsV2Command({ diff --git a/src/lib/server/bullmq/BullWorker.ts b/src/lib/server/bullmq/BullWorker.ts index 0afd3d0..9b3cd5b 100644 --- a/src/lib/server/bullmq/BullWorker.ts +++ b/src/lib/server/bullmq/BullWorker.ts @@ -1,10 +1,13 @@ -import type { Job } from 'bullmq'; +import { SpanStatusCode, trace } from '@opentelemetry/api'; +import type { Exception, Job } from 'bullmq'; import { Worker } from 'bullmq'; import * as Executor from '../job-executors'; import { getQueues, getWorkerConfig } from './queues'; import * as BullMQ from './types'; import { building } from '$app/environment'; +const tracer = trace.getTracer('BullWorker'); + export abstract class BullWorker { public worker?: Worker; constructor(public queue: BullMQ.QueueName) { @@ -13,11 +16,30 @@ export abstract class BullWorker { this.worker = new Worker(queue, this.runInternal.bind(this), getWorkerConfig()); } private async runInternal(job: Job) { - try { - return await this.run(job); - } catch (error) { - console.error(error); - } + return await tracer.startActiveSpan(`${job.queueName} - ${job.data.type}`, async (span) => { + span.setAttributes({ + 'job.id': job.id, + 'job.name': job.name, + 'job.queueName': job.queueName, + 'job.type': job.data.type, + 'job.opts': JSON.stringify(job.opts), + 'job.data': JSON.stringify(job.data) + }); + try { + job.updateProgress(0); + return await this.run(job); + } catch (error) { + span.recordException(error as Exception); + span.setStatus({ + code: SpanStatusCode.ERROR, // Error + message: (error as Error).message + }); + console.error(error); + throw error; + } finally { + span.end(); + } + }); } abstract run(job: Job): Promise; } diff --git a/src/lib/server/bullmq/queues.ts b/src/lib/server/bullmq/queues.ts index 5ff0b39..c34dffc 100644 --- a/src/lib/server/bullmq/queues.ts +++ b/src/lib/server/bullmq/queues.ts @@ -1,8 +1,10 @@ import { Queue } from 'bullmq'; +import { BullMQOtel } from 'bullmq-otel'; import { Redis } from 'ioredis'; import type { BuildJob, PollJob, ProjectJob, PublishJob, S3Job, SystemJob } from './types'; import { QueueName } from './types'; import { env } from '$env/dynamic/private'; +import OTEL from '$lib/otel'; class Connection { private conn: Redis; @@ -15,12 +17,22 @@ class Connection { }); this.connected = false; this.conn.on('close', () => { + OTEL.instance.logger.info('Valkey connection closed', { + isQueueConnection + }); this.connected = false; }); this.conn.on('connect', () => { + OTEL.instance.logger.info('Valkey connection established', { + isQueueConnection + }); this.connected = true; }); this.conn.on('error', (err) => { + OTEL.instance.logger.error('Valkey connection error', { + error: err.message, + isQueueConnection + }); this.connected = false; if (err.message.includes('ENOTFOUND')) { console.error('Fatal Valkey connection', err); @@ -41,6 +53,10 @@ class Connection { console.error(err); console.log('Valkey disconnected'); this.connected = false; + OTEL.instance.logger.error('Valkey disconnected', { + error: err.message, + isQueueConnection + }); } }); } @@ -78,7 +94,21 @@ export const getQueueConfig = () => { if (!_queueConnection) _queues = createQueues(); return { connection: _queueConnection!.connection(), - prefix: env.APP_ENV + '_build-engine' + prefix: env.APP_ENV + '_build-engine', + telemetry: new BullMQOtel(env.APP_ENV + '_build-engine'), + defaultJobOptions: { + // https://docs.bullmq.io/guide/queues/auto-removal-of-jobs#keep-a-certain-number-of-jobs + removeOnComplete: { + // 2 weeks + age: 2 * 7 * 24 * 60 * 60, + count: 1000 + }, + removeOnFail: { + // 2 weeks + age: 2 * 7 * 24 * 60 * 60, + count: 2000 + } + } } as const; }; let _queues: ReturnType | undefined = undefined; diff --git a/src/lib/server/job-executors/s3.ts b/src/lib/server/job-executors/s3.ts index e165a8c..b519d3a 100644 --- a/src/lib/server/job-executors/s3.ts +++ b/src/lib/server/job-executors/s3.ts @@ -32,7 +32,7 @@ export async function save(job: Job): Promise if (build.job) { s3.copyS3Folder(build); let defaultLanguage = await s3.readS3File(build, 'play-listing/default-language.txt'); - console.log(`getExtraContent defaultLanguage: ${defaultLanguage}`); + job.log(`getExtraContent defaultLanguage: ${defaultLanguage}`); const manifestFileContent = await s3.readS3File(build, 'manifest.txt'); let manifest: Record> = {}; if (manifestFileContent) { diff --git a/src/lib/server/prisma.ts b/src/lib/server/prisma.ts index a9d03ea..0048006 100644 --- a/src/lib/server/prisma.ts +++ b/src/lib/server/prisma.ts @@ -1,4 +1,5 @@ import { Prisma, PrismaClient } from '@prisma/client'; +import OTEL from '$lib/otel'; export const prisma = new PrismaClient(); @@ -14,6 +15,7 @@ class ConnectionChecker { await prisma.$queryRaw`SELECT 1`; if (!this.connected) { this.connected = true; + OTEL.instance.logger.info('Database connection established'); } } catch (e) { if ( @@ -24,6 +26,9 @@ class ConnectionChecker { // As best as I can tell, the only types of PrismaClientKnownRequestError that // should be thrown by the above query would involve the database being unreachable. if (this.connected) { + OTEL.instance.logger.error('Database connection lost', { + error: e.message + }); this.connected = false; console.log('Error checking database connection:', e); } diff --git a/src/routes/(auth)/exchange/+server.ts b/src/routes/(auth)/exchange/+server.ts index db3735d..88664e8 100644 --- a/src/routes/(auth)/exchange/+server.ts +++ b/src/routes/(auth)/exchange/+server.ts @@ -1,3 +1,4 @@ +import { trace } from '@opentelemetry/api'; import { error } from 'console'; import { EncryptJWT, jwtVerify } from 'jose'; import type { RequestHandler } from './$types'; @@ -16,6 +17,8 @@ export const GET: RequestHandler = async (event) => { throw error(400, 'Missing URL Search Params'); } + trace.getActiveSpan()?.setAttribute('request-id', requestId); + const verify = await getAuthConnection().get(requestId); if (!verify) { throw error(400, 'Invalid or expired code'); @@ -48,6 +51,8 @@ export const GET: RequestHandler = async (event) => { const token = await jwtVerify(res.id_token, key); + trace.getActiveSpan()?.setAttribute('user.email', token.payload.email as string); + const encryptedToken = await new EncryptJWT(token.payload) .setProtectedHeader({ alg: 'dir', enc: 'A256CBC-HS512' }) .encrypt(key);