build: Added java 21 build with maven and NVD libraries

2026-04-14 12:27:41 +08:00
commit e75d014023
4 changed files with 256 additions and 0 deletions
@@ -0,0 +1,100 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "ci/**"
+  schedule:
+    - cron: "0 10 * * 0"
+  workflow_dispatch:
+
+env:
+  HARBOR_REGISTRY: 192.168.1.72
+  HARBOR_PROJECT: kollect-tools
+
+jobs:
+  # Detect which images changed
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      has_changes: ${{ steps.set-matrix.outputs.has_changes }}
+    steps:
+      - uses: https://github.com/actions/checkout@v6
+        with:
+          fetch-depth: 2
+
+      - name: Detect changed images
+        id: set-matrix
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            # On manual trigger, build all images
+            IMAGES=$(find ci -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          else
+            # On push, only build changed images
+            IMAGES=$(git diff --name-only HEAD~1 HEAD -- ci/ \
+              | cut -d'/' -f2 | sort -u \
+              | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          fi
+
+          if [ "$IMAGES" = "[]" ] || [ -z "$IMAGES" ]; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+            echo "matrix={\"image\":[]}" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "matrix={\"image\":$IMAGES}" >> "$GITHUB_OUTPUT"
+          fi
+
+  build-and-push:
+    needs: detect-changes
+    if: needs.detect-changes.outputs.has_changes == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJson(needs.detect-changes.outputs.matrix) }}
+    steps:
+      - uses: https://github.com/actions/checkout@v6
+      - name: Set image metadata
+        id: meta
+        run: |
+          IMAGE_NAME="${{ env.HARBOR_REGISTRY }}/${{ env.HARBOR_PROJECT }}/${{ matrix.image }}"
+          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+          TIMESTAMP=$(date +%Y%m%d)
+
+          echo "image_name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "tag_sha=${IMAGE_NAME}:${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          echo "tag_date=${IMAGE_NAME}:${TIMESTAMP}" >> "$GITHUB_OUTPUT"
+          echo "tag_latest=${IMAGE_NAME}:latest" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Harbor
+        run: |
+          echo "${{ secrets.HARBOR_PASSWORD }}" \
+            | docker login "${{ env.HARBOR_REGISTRY }}" \
+                -u "${{ secrets.HARBOR_USERNAME }}" \
+                --password-stdin
+
+      - name: Build image
+        run: |
+          docker build \
+            --label "org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}" \
+            --label "org.opencontainers.image.revision=${{ github.sha }}" \
+            -t "${{ steps.meta.outputs.tag_sha }}" \
+            -t "${{ steps.meta.outputs.tag_date }}" \
+            -t "${{ steps.meta.outputs.tag_latest }}" \
+            "ci/${{ matrix.image }}/"
+
+      - name: Push image
+        run: |
+          docker push "${{ steps.meta.outputs.tag_sha }}"
+          docker push "${{ steps.meta.outputs.tag_date }}"
+          docker push "${{ steps.meta.outputs.tag_latest }}"
+
+      - name: Summary
+        run: |
+          echo "### Pushed image" >> "$GITHUB_STEP_SUMMARY"
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "| Tag | Value |" >> "$GITHUB_STEP_SUMMARY"
+          echo "|-----|-------|" >> "$GITHUB_STEP_SUMMARY"
+          echo "| latest | \`${{ steps.meta.outputs.tag_latest }}\` |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| sha | \`${{ steps.meta.outputs.tag_sha }}\` |" >> "$GITHUB_STEP_SUMMARY"
+          echo "| date | \`${{ steps.meta.outputs.tag_date }}\` |" >> "$GITHUB_STEP_SUMMARY"
@@ -0,0 +1,38 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Purpose
+
+This repo holds CI Docker images for KollectAI infrastructure. Each subdirectory under `ci/` is a self-contained image with its own Dockerfile and README.
+
+## Architecture
+
+- **`ci/<image-name>/`** — One directory per Docker image (e.g. `ci/java-builder/`). Each contains a Dockerfile and a README with build args and usage.
+- **`.gitea/workflows/build-and-push.yaml`** — Gitea Actions workflow that auto-detects which images changed and builds/pushes only those. On schedule or manual dispatch, it builds all images.
+
+## Registries
+
+- **Harbor**: `192.168.1.72/kollect-tools/<image-name>` — production image store (project: `kollect-tools`)
+- Images are tagged three ways: `latest`, short commit SHA, and `YYYYMMDD` date
+
+## CI Workflow
+
+The workflow triggers on:
+- Push to `main` affecting `ci/**` (builds only changed images)
+- Weekly schedule (Sunday 10:00 UTC = Monday 00:00 UTC+14)
+- Manual dispatch (builds all images)
+
+Required Gitea secrets: `HARBOR_USERNAME`, `HARBOR_PASSWORD`
+
+## Adding a New Image
+
+1. Create `ci/<image-name>/Dockerfile`
+2. Add a `ci/<image-name>/README.md` documenting build args and usage
+3. Push to `main` — the CI workflow auto-discovers it via `find ci -mindepth 1 -maxdepth 1 -type d`
+
+## Building Locally
+
+```bash
+docker build -t 192.168.1.72/kollect-tools/<image-name>:latest ci/<image-name>/
+```
@@ -0,0 +1,65 @@
+# KollectAI CI — Java Builder Image
+#
+# Pre-baked build environment for backend + plugin CI jobs.
+# Contains: Java 21, Maven 3.9.9, common dependencies, OWASP NVD database.
+#
+# Build:
+#   docker build -t registry.kollect.biz/kollect-ci/java-builder:latest ci/java-builder/
+#
+# Usage in CI:
+#   container:
+#     image: registry.kollect.biz/kollect-ci/java-builder:latest
+
+ARG JAVA_VERSION=21
+FROM eclipse-temurin:${JAVA_VERSION}-jdk-jammy
+
+ARG MAVEN_VERSION=3.9.9
+ARG OWASP_DC_VERSION=12.1.1
+# ─────────────────────────────────────────────────────────────────────
+# System dependencies
+# ─────────────────────────────────────────────────────────────────────
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        git \
+        jq \
+        unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# ─────────────────────────────────────────────────────────────────────
+# Maven
+# ─────────────────────────────────────────────────────────────────────
+ENV MAVEN_HOME=/opt/maven
+ENV PATH="${MAVEN_HOME}/bin:${PATH}"
+
+RUN curl -fsSL "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" \
+    | tar -xz -C /opt \
+    && mv "/opt/apache-maven-${MAVEN_VERSION}" "${MAVEN_HOME}"
+
+# ─────────────────────────────────────────────────────────────────────
+# OWASP Dependency-Check — pre-download NVD database
+#
+# This is the single biggest CI time saver. The NVD database download
+# takes 5-10 minutes on a cold cache. Pre-baking it into the image
+# means audit jobs start with a warm database.
+#
+# Rebuild this image weekly to keep the NVD database fresh.
+# ─────────────────────────────────────────────────────────────────────
+ENV OWASP_DATA_DIR=/opt/owasp/dependency-check-data
+
+RUN mkdir -p "${OWASP_DATA_DIR}" \
+    && mvn org.owasp:dependency-check-maven:${OWASP_DC_VERSION}:update-only \
+        -DdataDirectory="${OWASP_DATA_DIR}" \
+        -q || true
+
+# ─────────────────────────────────────────────────────────────────────
+# ORAS CLI — for uploading artifacts to Harbor
+# ─────────────────────────────────────────────────────────────────────
+ARG ORAS_VERSION=1.2.2
+
+RUN curl -fsSL "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" \
+    | tar -xz -C /usr/local/bin oras
+
+WORKDIR /workspace
+
+# Verify installation
+RUN java -version && mvn -version && oras version && jq --version
@@ -0,0 +1,53 @@
+# Java Builder — CI Image
+
+Pre-baked build environment for KollectAI-ETL backend and plugin CI jobs.
+
+## What's included
+
+- Java 21 (Eclipse Temurin)
+- Maven 3.9.9
+- Pre-cached Maven dependencies (Spring Boot, Flink, MyBatis, etc.)
+- Pre-installed `plugin-api` in local Maven repo
+- OWASP NVD database snapshot
+- ORAS CLI (Harbor artifact uploads)
+- git, jq, curl
+
+## Build
+
+```bash
+docker build -t registry.kollect.biz/kollect-ci/java-builder:latest ci/java-builder/
+docker push registry.kollect.biz/kollect-ci/java-builder:latest
+```
+
+### Build args
+
+| Arg | Default | Description |
+|-----|---------|-------------|
+| `MAVEN_VERSION` | `3.9.9` | Maven version |
+| `OWASP_DC_VERSION` | `12.1.1` | OWASP Dependency-Check version |
+| `ETL_BRANCH` | `001-ai-etl-platform` | Branch to fetch pom.xml files from |
+| `ORAS_VERSION` | `1.2.2` | ORAS CLI version |
+
+## Usage in CI
+
+```yaml
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    container:
+      image: registry.kollect.biz/kollect-ci/java-builder:latest
+    steps:
+      - uses: actions/checkout@v6
+      - run: ./mvnw -f backend/etl/pom.xml test -Dgroups=unit -q
+```
+
+## Maintenance
+
+Rebuild weekly to keep the OWASP NVD database fresh:
+
+```bash
+docker build --no-cache -t registry.kollect.biz/kollect-ci/java-builder:latest ci/java-builder/
+docker push registry.kollect.biz/kollect-ci/java-builder:latest
+```
+
+When `pom.xml` files change (new dependencies), rebuild to update the cached deps layer.