feat: implement rubygems registry [CM-1241]

scripts/builders/packages.env

@@ -1,4 +1,4 @@
 DOCKERFILE="./services/docker/Dockerfile.packages"
 CONTEXT="../"
 REPO="sjc.ocir.io/axbydjxa5zuh/packages"
-SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker"
+SERVICES="github-repos-enricher bq-dataset-ingest npm-worker pypi-worker maven-worker osv-worker dockerhub-sync cargo-worker go-worker nuget-worker security-contacts-worker rubygems-worker"

scripts/services/rubygems-worker.yaml

@@ -0,0 +1,65 @@
+version: '3.1'
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: rubygems-worker
+  SHELL: /bin/sh
+  SUPPRESS_NO_CONFIG_WARNING: 'true'
+  CROWD_TEMPORAL_TASKQUEUE: rubygems-worker
+
+services:
+  rubygems-worker:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run start:rubygems-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+
+  rubygems-worker-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.packages
+    command: 'pnpm run dev:rubygems-worker'
+    working_dir: /usr/crowd/app/services/apps/packages_worker
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: rubygems-worker
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src
+      - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src
+      - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src
+      - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src
+      - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src
+      - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src
+      - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src
+      - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src
+      - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src
+      - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src
+      - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src
+      - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src
+      - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src
+      - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src
+      - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src
+      - ../../services/apps/packages_worker/src:/usr/crowd/app/services/apps/packages_worker/src
+
+networks:
+  crowd-bridge:
+    external: true

services/apps/packages_worker/package.json

@@ -49,6 +49,9 @@
     "start:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker tsx src/bin/nuget-worker.ts",
     "dev:nuget-worker": "CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
     "dev:nuget-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=nuget-worker SERVICE=nuget-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9242 src/bin/nuget-worker.ts",
+    "start:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker tsx src/bin/rubygems-worker.ts",
+    "dev:rubygems-worker": "CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
+    "dev:rubygems-worker:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=rubygems-worker SERVICE=rubygems-worker LOG_LEVEL=trace nodemon --watch src --watch ../../libs --ext ts --exec tsx --inspect=0.0.0.0:9244 src/bin/rubygems-worker.ts",
     "backfill:maven": "SERVICE=maven tsx src/bin/maven-backfill.ts",
     "backfill:maven:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=maven LOG_LEVEL=info tsx src/bin/maven-backfill.ts",
     "import:maven-maintainers": "SERVICE=maven tsx src/maven/scripts/importMaintainersFromCsv.ts",

services/apps/packages_worker/src/activities.ts

@@ -32,6 +32,11 @@ export {
 } from './pypi/activities'
 export { getCriticalPypiCount } from './pypi/downloads/getCriticalPypiCount'
 export { processNuGetBatch } from './nuget/activities'
+export {
+  processRubyGemsCoreBatch,
+  processRubyGemsCriticalBatch,
+  processRubyGemsDependentsBatch,
+} from './rubygems/activities'
 export {
   processSecurityContactsBatch,
   ingestSecurityContactsForPurlActivity,

services/apps/packages_worker/src/bin/rubygems-worker.ts

@@ -0,0 +1,14 @@
+import {
+  scheduleRubyGemsCriticalIngestion,
+  scheduleRubyGemsDependentsIngestion,
+  scheduleRubyGemsIngestion,
+} from '../rubygems/schedule'
+import { svc } from '../service'
+
+setImmediate(async () => {
+  await svc.init()
+  await scheduleRubyGemsIngestion()
+  await scheduleRubyGemsCriticalIngestion()
+  await scheduleRubyGemsDependentsIngestion()
+  await svc.start()
+})

services/apps/packages_worker/src/config.ts

@@ -86,6 +86,27 @@ export function getNuGetConfig() {
   }
 }
 
+export function getRubyGemsConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_FETCHER_BATCH_SIZE ?? '10000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_FETCHER_CONCURRENCY ?? '8', 10),
+  }
+}
+
+export function getRubyGemsCriticalConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_BATCH_SIZE ?? '5000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_CRITICAL_FETCHER_CONCURRENCY ?? '4', 10),
+  }
+}
+
+export function getRubyGemsDependentsConfig() {
+  return {
+    batchSize: parseInt(process.env.RUBYGEMS_DEPENDENTS_BATCH_SIZE ?? '10000', 10),
+    concurrency: parseInt(process.env.RUBYGEMS_DEPENDENTS_CONCURRENCY ?? '50', 10),
+  }
+}
+
 export function getDockerhubConfig() {
   return {
     hubBaseUrl: requireEnv('DOCKERHUB_API_BASE_URL'),

services/apps/packages_worker/src/rubygems/activities.ts

@@ -0,0 +1,47 @@
+import { getServiceChildLogger } from '@crowd/logging'
+
+import {
+  getRubyGemsConfig,
+  getRubyGemsCriticalConfig,
+  getRubyGemsDependentsConfig,
+} from '../config'
+import { getPackagesDb } from '../db'
+
+import { processBatch as processCoreBatch } from './runRubyGemsCoreLoop'
+import { processBatch as processCriticalBatch } from './runRubyGemsCriticalLoop'
+import {
+  DependentsBatchResult,
+  processBatch as processDependentsBatch,
+} from './runRubyGemsDependentsLoop'
+import { BatchResult } from './types'
+
+const log = getServiceChildLogger('rubygems-activity')
+
+export async function processRubyGemsCoreBatch(): Promise<BatchResult> {
+  const config = getRubyGemsConfig()
+  const qx = await getPackagesDb()
+  const today = new Date().toISOString().split('T')[0]
+  const result = await processCoreBatch(qx, config, today)
+  log.info({ ...result }, 'RubyGems core batch complete')
+  return result
+}
+
+export async function processRubyGemsCriticalBatch(
+  afterId = '0',
+): Promise<BatchResult & { lastId: string | null }> {
+  const config = getRubyGemsCriticalConfig()
+  const qx = await getPackagesDb()
+  const result = await processCriticalBatch(qx, config, afterId)
+  log.info({ ...result }, 'RubyGems critical batch complete')
+  return result
+}
+
+export async function processRubyGemsDependentsBatch(
+  afterId = '0',
+): Promise<DependentsBatchResult> {
+  const config = getRubyGemsDependentsConfig()
+  const qx = await getPackagesDb()
+  const result = await processDependentsBatch(qx, config, afterId)
+  log.info({ ...result }, 'RubyGems dependents batch complete')
+  return result
+}

services/apps/packages_worker/src/rubygems/client.ts

@@ -0,0 +1,60 @@
+import axios from 'axios'
+
+import { acquireRubyGemsSlot, parseRetryAfterMs } from './rateLimiter'
+import {
+  RubyGemsFetchResult,
+  RubyGemsGemResponse,
+  RubyGemsOwner,
+  RubyGemsVersionItem,
+} from './types'
+
+const MAX_RATE_LIMIT_RETRIES = 5
+
+async function rubyGemsGet<T>(url: string): Promise<RubyGemsFetchResult<T>> {
+  for (let attempt = 0; ; attempt++) {
+    await acquireRubyGemsSlot()
+    try {
+      const resp = await axios.get<T>(url, { timeout: 15000 })
+      return resp.data
+    } catch (err) {
+      if (axios.isAxiosError(err)) {
+        const status = err.response?.status
+        if (status === 404) return { kind: 'NOT_FOUND', status, message: err.message }
+        if (status === 429) {
+          if (attempt >= MAX_RATE_LIMIT_RETRIES) {
+            return { kind: 'RATE_LIMIT', status, message: err.message }
+          }
+          await new Promise((r) =>
+            setTimeout(r, parseRetryAfterMs(err.response?.headers['retry-after'])),
+          )
+          continue
+        }
+      }
+      throw err
+    }
+  }
+}
+
+export function fetchGem(name: string): Promise<RubyGemsFetchResult<RubyGemsGemResponse>> {
+  return rubyGemsGet<RubyGemsGemResponse>(
+    `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}.json`,
+  )
+}
+
+export function fetchVersions(name: string): Promise<RubyGemsFetchResult<RubyGemsVersionItem[]>> {
+  return rubyGemsGet<RubyGemsVersionItem[]>(
+    `https://rubygems.org/api/v1/versions/${encodeURIComponent(name)}.json`,
+  )
+}
+
+export function fetchOwners(name: string): Promise<RubyGemsFetchResult<RubyGemsOwner[]>> {
+  return rubyGemsGet<RubyGemsOwner[]>(
+    `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/owners.json`,
+  )
+}
+
+export function fetchReverseDependencies(name: string): Promise<RubyGemsFetchResult<string[]>> {
+  return rubyGemsGet<string[]>(
+    `https://rubygems.org/api/v1/gems/${encodeURIComponent(name)}/reverse_dependencies.json`,
+  )
+}

services/apps/packages_worker/src/rubygems/normalize.ts

@@ -0,0 +1,58 @@
+import {
+  NormalizedRubyGemsOwner,
+  NormalizedRubyGemsPackage,
+  NormalizedRubyGemsVersion,
+  RubyGemsGemResponse,
+  RubyGemsOwner,
+  RubyGemsVersionItem,
+} from './types'
+
+function nonEmpty(value: string | null | undefined): string | null {
+  if (!value) return null
+  const trimmed = value.trim()
+  return trimmed === '' ? null : trimmed
+}
+
+export function normalizeRubyGemsPackage(doc: RubyGemsGemResponse): NormalizedRubyGemsPackage {
+  const licenses = doc.licenses && doc.licenses.length > 0 ? doc.licenses : null
+  return {
+    description: nonEmpty(doc.info),
+    homepage: nonEmpty(doc.homepage_uri),
+    declaredRepositoryUrl: nonEmpty(doc.source_code_uri),
+    licenses,
+    licensesRaw: licenses ? licenses.join(', ') : null,
+    latestVersion: nonEmpty(doc.version),
+    totalDownloads: doc.downloads ?? 0,
+  }
+}
+
+function parseCreatedAt(value: string | undefined): Date | null {
+  if (!value) return null
+  const date = new Date(value)
+  return isNaN(date.getTime()) ? null : date
+}
+
+export function normalizeRubyGemsVersions(
+  items: RubyGemsVersionItem[],
+): NormalizedRubyGemsVersion[] {
+  return items.map((item) => ({
+    number: item.number,
+    publishedAt: parseCreatedAt(item.created_at),
+    isPrerelease: item.prerelease ?? false,
+    licenses: item.licenses && item.licenses.length > 0 ? item.licenses : null,
+  }))
+}
+
+export function pickLatestRubyGemsVersion(
+  versions: NormalizedRubyGemsVersion[],
+): NormalizedRubyGemsVersion | null {
+  if (versions.length === 0) return null
+  const stable = versions.find((v) => !v.isPrerelease)
+  return stable ?? versions[0]
+}
+
+export function normalizeRubyGemsOwners(owners: RubyGemsOwner[]): NormalizedRubyGemsOwner[] {
+  return owners
+    .filter((o): o is RubyGemsOwner & { handle: string } => !!o.handle && o.handle.trim() !== '')
+    .map((o) => ({ username: o.handle, email: nonEmpty(o.email) }))
+}

services/apps/packages_worker/src/rubygems/rateLimiter.ts

@@ -0,0 +1,22 @@
+const MAX_RPS = Math.max(1, parseInt(process.env.RUBYGEMS_MAX_RPS ?? '10', 10))
+const INTERVAL_MS = 1000 / MAX_RPS
+
+let nextSlot = 0
+
+export async function acquireRubyGemsSlot(): Promise<void> {
+  const now = Date.now()
+  const slot = Math.max(now, nextSlot)
+  nextSlot = slot + INTERVAL_MS
+  const wait = slot - now
+  if (wait > 0) await new Promise((r) => setTimeout(r, wait))
+}
+
+export function parseRetryAfterMs(header: unknown): number {
+  const FALLBACK_MS = 1000
+  if (typeof header !== 'string') return FALLBACK_MS
+  const seconds = Number(header)
+  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000)
+  const date = new Date(header)
+  if (!Number.isNaN(date.getTime())) return Math.max(0, date.getTime() - Date.now())
+  return FALLBACK_MS
+}