diff --git a/AGENTS.md b/AGENTS.md index 6b67851..b9e05ac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -743,7 +743,7 @@ For a normal code change run the relevant focused tests plus: - `npm run build:web` - `npm run typecheck:scripts` - `npx tsc --noEmit -p tsconfig.next.json` -- `make check` before every push (see Tests & Security Baseline) +- `make prepush` before every push, full `make check` before deploy and weekly (see Tests & Security Baseline) Use a concise imperative commit message for each completed, verified work package. @@ -758,6 +758,8 @@ Scanners (gitleaks, Semgrep, Trivy) run as Docker images; npm runs locally or in |---------|--------------| | `make quick` | `npm run lint` (ESLint security rules) + `npm run typecheck` + `npm test` | | `make check` | secrets (gitleaks) + quick + Semgrep (`.semgrep/`) + `npm audit --audit-level=high` | +| `make prepush` | before every push: only the checks the diff needs (`scripts/security/changed_scope.py`: docs -> secrets only, code -> lint/test/SAST on changed files, deps -> + audit, auth/routes/schemas -> + mutation, baseline tooling -> everything) | +| `make weekly` | weekly cron on cj-ki (Mon 05:00): `make check` + Trivy, notifies only on red | | `make mutation` | `security-mutations.toml`: removes one protection at a time, every mutation must turn a test red | | `make vet-deps PKGS="npm:name"` | checks a new package before it is added | | `make container IMAGE=…`, `make zap TARGET=…` | image/Dockerfile scan, ZAP baseline against an own instance | @@ -787,7 +789,7 @@ Rules for agents working on this repo: - Input only through zod schemas with `.strict()` (reject unknown fields); SQL only via Drizzle or bound parameters; no secrets in code, repo, logs or chat. - New packages only after `dependency-vetting` / `make vet-deps`. -- Before every push `make check`; red means no push. A diff touching auth, roles, routes +- Before every push `make prepush` (runs only the checks the diff needs; full `make check` before deploy and weekly via cron); red means no push. A diff touching auth, roles, routes or personal data gets a `security-review` in a fresh context. Before a deploy run `pre-release-gate`. - Report checks that did not run as "not run", never as passed. diff --git a/Makefile b/Makefile index 6d2236e..d9296fa 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,12 @@ # # Werkzeuge: Python-Tools im venv der App, Scanner (gitleaks, Semgrep, Trivy, hadolint, ZAP, # schemathesis) als Docker-Images, npm lokal oder im node-Container. +# +# Wann was läuft: +# beim Coden make quick (Lint + Tests; TESTS=... nur betroffene + Sicherheitstests) +# vor dem Push make prepush (nur was der Diff braucht, siehe scripts/security/changed_scope.py) +# vor Deploy make check (alles, über Skill pre-release-gate) +# wöchentlich make weekly (check + Trivy, zeitabhängige Funde wie neue CVEs) -include baseline.mk @@ -17,24 +23,49 @@ NODE_IMAGE ?= node:24 OPENAPI_PATH ?= /openapi.json FUZZ_CHECKS ?= not_a_server_error ZAP_ARGS ?= -I +# prepush vergleicht gegen BASE (leer = Upstream des Branches, sonst origin/HEAD bzw. main). +BASE ?= +# make quick TESTS="tests/test_x.py": nur diese Python-Tests (relativ zu PY_DIR) plus Sicherheitstests. +TESTS ?= +PY_SECURITY_TESTS ?= $(patsubst $(PY_DIR)/%,%,$(wildcard $(PY_DIR)/tests/test_security_baseline.py)) +TRIVY_CACHE ?= $(HOME)/.cache/trivy +# Standardquelle mirror.gcr.io lieferte am 16.10.2026 nur 404, ghcr.io funktioniert. +TRIVY_DB ?= ghcr.io/aquasecurity/trivy-db:2 +# Einteilung des Diffs für prepush (reguläre Ausdrücke, leer = Standard aus changed_scope.py). +SECURITY_PATHS ?= +DOCS_PATHS ?= +DEPS_PATHS ?= +TOOLING_PATHS ?= +export SECURITY_PATHS DOCS_PATHS DEPS_PATHS TOOLING_PATHS MUTATIONS GITLEAKS_IMAGE ?= ghcr.io/gitleaks/gitleaks:v8.30.1 SEMGREP_IMAGE ?= semgrep/semgrep:1.179.0 -TRIVY_IMAGE ?= aquasec/trivy:latest -HADOLINT_IMAGE ?= hadolint/hadolint:latest -ZAP_IMAGE ?= ghcr.io/zaproxy/zaproxy:stable -SCHEMATHESIS_IMAGE ?= schemathesis/schemathesis:stable +# Scanner mit Tag UND Digest pinnen: ein umgehängter Tag (Trivy-Kompromittierung 03/2026, +# GHSA-69fq-xp46-6x23) kommt so nicht still herein. Wichtig vor allem für Trivy, das in +# `make container` den Docker-Socket bekommt (= root auf dem Host). Aktualisieren: +# docker buildx imagetools inspect : --format '{{.Manifest.Digest}}' +# und hier wie in ci/security.yml eintragen; neue Versionen erst nach ~2 Wochen übernehmen. +TRIVY_IMAGE ?= aquasec/trivy:0.75.0@sha256:af6acf9a6b85dfe389a1941505c0ce9efef52a4719635e1a962f022a3d855daa +HADOLINT_IMAGE ?= hadolint/hadolint:v2.15.1@sha256:32dac94127fd60b7b7e3fbfc65e1383b9b5e25c9bfd7b8536de7a539fe68a12d +ZAP_IMAGE ?= ghcr.io/zaproxy/zaproxy:2.17.0@sha256:7aaa659b0d43078febd82e29bad112285c370727e86ab8340444220e17d9f0d2 +SCHEMATHESIS_IMAGE ?= schemathesis/schemathesis:v4.31.0@sha256:58a4ffa68cd690a04eb70fd2d1c790acd54cd73986c777ec2556c901b6b707e4 HAS_PY := $(wildcard $(PY_DIR)/requirements.txt $(PY_DIR)/pyproject.toml) HAS_NODE := $(wildcard $(NODE_DIR)/package.json) VENV := $(PY_DIR)/.venv TOOLS := .venv-tools UID_GID := $(shell id -u):$(shell id -g) -DOCKER := docker run --rm -u $(UID_GID) -e HOME=/tmp -v "$(CURDIR)":/src -w /src +# In einem git-Worktree ist .git nur ein Verweis auf das Haupt-Repo. Ohne dieses im Container +# findet gitleaks keine Commits und meldet trotzdem "no leaks found" (grün ohne Prüfung), und +# Semgrep kennt die .gitignore nicht und scannt Build-Ausgaben wie .next/. +GIT_COMMON := $(shell git rev-parse --path-format=absolute --git-common-dir 2>/dev/null) +GIT_MOUNT := $(if $(filter-out $(CURDIR)/.git,$(GIT_COMMON)),-v "$(GIT_COMMON)":"$(GIT_COMMON)":ro) +DOCKER := docker run --rm -u $(UID_GID) -e HOME=/tmp -v "$(CURDIR)":/src $(GIT_MOUNT) -w /src +GITLEAKS :=docker run --rm -u $(UID_GID) -v "$(CURDIR)":/repo $(GIT_MOUNT) $(GITLEAKS_IMAGE) git /repo --redact --no-banner $(if $(wildcard .gitleaks.toml),-c /repo/.gitleaks.toml) NPM ?= $(if $(shell command -v npm 2>/dev/null),npm --prefix $(NODE_DIR),docker run --rm -u $(UID_GID) -e HOME=/tmp -v "$(CURDIR)":/w -w /w/$(NODE_DIR) $(NODE_IMAGE) npm) .DEFAULT_GOAL := help -.PHONY: help setup quick check lint test secrets sast audit container zap fuzz mutation mutmut vet-deps +.PHONY: help setup quick prepush check weekly lint test secrets secrets-diff sast sast-diff audit trivy-fs container zap fuzz mutation mutmut vet-deps help: ## Diese Übersicht @grep -hE '^[a-z-]+:.*## ' $(MAKEFILE_LIST) | awk -F':.*## ' '{printf " make %-12s %s\n", $$1, $$2}' @@ -52,11 +83,19 @@ endif $(TOOLS)/bin/pre-commit install @echo "Setup fertig. Weiter mit: make quick" -quick: lint test ## Lint + Tests (Sekunden, nach jeder Änderung) +quick: lint test ## Lint + Tests beim Coden; nur betroffene: make quick TESTS="tests/test_x.py" -check: secrets lint test sast audit ## Alles vor dem Push: Secrets, Lint, Tests, SAST, Dependency-Audit +prepush: ## Vor dem Push: nur die Prüfungen, die der Diff braucht (BASE=ref setzt die Basis) + @python3 $(SCRIPTS)/changed_scope.py explain $(BASE) + @t=$$(python3 $(SCRIPTS)/changed_scope.py targets $(BASE)) && $(MAKE) --no-print-directory $$t && echo "make prepush: grün ($$t)" + +check: secrets lint test sast audit ## Alles: Secrets, Lint, Tests, SAST, Audit (vor Deploy, bei Werkzeug-Änderung) @echo "make check: alles grün" +weekly: ## Wochenlauf: check + Trivy, läuft nach Fehlern weiter (neue CVEs kommen ohne Code-Änderung) + $(MAKE) -k --no-print-directory secrets lint test sast audit trivy-fs + @echo "make weekly: alles grün" + lint: ## Ruff (Python), ESLint + Typecheck (Node, falls als npm-Skript vorhanden) ifneq ($(HAS_PY),) cd $(PY_DIR) && .venv/bin/ruff check . && .venv/bin/ruff format --check . @@ -68,14 +107,19 @@ endif test: ## Unit-, Integrations- und Sicherheitstests ifneq ($(HAS_PY),) - cd $(PY_DIR) && .venv/bin/python -m pytest -q + cd $(PY_DIR) && .venv/bin/python -m pytest -q $(if $(TESTS),$(TESTS) $(PY_SECURITY_TESTS)) endif ifneq ($(HAS_NODE),) $(NPM) test endif -secrets: ## gitleaks über die Git-Historie - docker run --rm -u $(UID_GID) -v "$(CURDIR)":/repo $(GITLEAKS_IMAGE) git /repo --redact --no-banner $(if $(wildcard .gitleaks.toml),-c /repo/.gitleaks.toml) +secrets: ## gitleaks über die ganze Git-Historie + $(GITLEAKS) + +secrets-diff: ## gitleaks nur über die Commits seit BASE (offene Änderungen fängt der pre-commit-Hook) + @range=$$(python3 $(SCRIPTS)/changed_scope.py range $(BASE)); \ + if [ -z "$$range" ]; then echo "secrets-diff: keine Basis gefunden, prüfe die ganze Historie"; $(GITLEAKS); \ + else $(GITLEAKS) --log-opts="$$range"; fi sast: ## Semgrep (eigene Regeln) + Bandit $(DOCKER) $(SEMGREP_IMAGE) semgrep scan --config $(SEMGREP_RULES) --error --metrics=off --quiet @@ -83,6 +127,15 @@ ifneq ($(HAS_PY),) $(VENV)/bin/bandit -q -r $(PY_DIR)/$(PY_SRC) endif +PY_PREFIX := $(if $(filter .,$(PY_DIR)),./,./$(PY_DIR)/) +sast-diff: ## Semgrep + Bandit nur über die geänderten Dateien (vor dem Push) + @files=$$(python3 $(SCRIPTS)/changed_scope.py files $(BASE) | sed 's|^|./|'); \ + if [ -z "$$files" ]; then echo "sast-diff: keine geänderten Dateien"; exit 0; fi; \ + printf '%s\n' "$$files" | xargs -d '\n' $(DOCKER) $(SEMGREP_IMAGE) semgrep scan --config $(SEMGREP_RULES) --error --metrics=off --quiet || exit 1; \ + [ -n "$(HAS_PY)" ] || exit 0; \ + py=$$(printf '%s\n' "$$files" | awk -v p='$(PY_PREFIX)' 'index($$0, p) == 1 && /\.py$$/ && !/\/(tests?|\.venv[^/]*|node_modules)\//'); \ + if [ -n "$$py" ]; then printf '%s\n' "$$py" | xargs -d '\n' $(VENV)/bin/bandit -q; fi + audit: ## Bekannte Lücken in Abhängigkeiten (pip-audit, npm audit) ifneq ($(HAS_PY),) $(VENV)/bin/pip-audit -r $(PY_DIR)/requirements.txt @@ -91,6 +144,10 @@ ifneq ($(HAS_NODE),) $(NPM) audit --audit-level=high endif +trivy-fs: ## Trivy übers Repo: Abhängigkeiten, Secrets, Fehlkonfiguration (HIGH/CRITICAL mit Fix) + @mkdir -p "$(TRIVY_CACHE)" + $(DOCKER) -v "$(TRIVY_CACHE)":/cache $(TRIVY_IMAGE) fs --cache-dir /cache --db-repository $(TRIVY_DB) --quiet --scanners vuln,secret,misconfig --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 --skip-dirs '**/node_modules' --skip-dirs '**/.venv' --skip-dirs '**/.venv-tools' . + container: ## Image + Dockerfile/Compose scannen: make container IMAGE=name:tag @test -n "$(IMAGE)" || (echo "IMAGE=name:tag angeben" && exit 2) docker run --rm -v /var/run/docker.sock:/var/run/docker.sock $(TRIVY_IMAGE) image --severity HIGH,CRITICAL --exit-code 1 --ignore-unfixed $(IMAGE) diff --git a/scripts/security/changed_scope.py b/scripts/security/changed_scope.py new file mode 100644 index 0000000..589ff64 --- /dev/null +++ b/scripts/security/changed_scope.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Leitet aus dem Git-Diff ab, welche Baseline-Prüfungen vor dem Push nötig sind. + +Aufruf (aus dem Repo-Root, so wie das Makefile es tut): + changed_scope.py targets [BASE] make-Ziele für `make prepush`, eine Zeile + changed_scope.py files [BASE] geänderte, noch vorhandene Dateien (für `make sast-diff`) + changed_scope.py range [BASE] Commit-Bereich für gitleaks, leer = keine Basis gefunden + changed_scope.py explain [BASE] Begründung für Menschen + +BASE ist die Git-Referenz, gegen die verglichen wird. Ohne BASE: Upstream des Branches, sonst +origin/HEAD, origin/main, origin/master, main, master. Findet sich keine Basis, wird alles +geprüft (sicherer Standard). + +Geprüft wird der Unterschied zwischen Basis und Arbeitsverzeichnis: die Commits seit der Basis +plus noch nicht committete und neue Dateien. + +Einteilung (Reihenfolge zählt, erste passende Regel gewinnt): + Werkzeug Makefile, baseline.mk, Semgrep-Regeln, Mutationen, pre-commit, CI -> volles check + Abhängigk. requirements*, pyproject, package*.json, Lockfiles -> lint test audit + ohne Code Doku, Bilder -> nur Secrets + Code alles andere -> lint test sast-diff +Zusätzlich: Pfad passt auf SECURITY_PATHS (Auth, Rollen, Routen, Schemas ...) -> + mutation. + +Anpassbar per Umgebung (in baseline.mk setzen, das Makefile exportiert sie): + SECURITY_PATHS, DOCS_PATHS, DEPS_PATHS, TOOLING_PATHS (reguläre Ausdrücke), MUTATIONS (Datei). +""" + +import os +import re +import shutil +import subprocess # nosec B404 - nur feste git-Aufrufe ohne Shell +import sys + +GIT = shutil.which("git") or "git" + +DEFAULT_TOOLING = ( + r"(^|/)(Makefile|baseline\.mk|\.pre-commit-config\.yaml|\.gitleaks\.toml|" + r"[^/]*mutations[^/]*\.toml|\.semgrepignore)$|(^|/)\.semgrep/|(^|/)scripts/security/|" + r"(^|/)\.forgejo/|(^|/)\.github/workflows/" +) +DEFAULT_DEPS = ( + r"(^|/)(requirements[^/]*\.(txt|in)|pyproject\.toml|poetry\.lock|uv\.lock|" + r"Pipfile(\.lock)?|package(-lock)?\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock)$" +) +DEFAULT_DOCS = ( + r"(\.(md|rst|adoc|txt|png|jpe?g|gif|webp|ico|pdf)$|^docs/|(^|/)LICENSE[^/]*$|" + r"(^|/)\.gitignore$|(^|/)\.gitattributes$)" +) +DEFAULT_SECURITY = ( + r"(?i)(auth|login|logout|session|token|passw|secret|permission|role|rbac|acl|owner|" + r"tenant|route|router|middleware|schema|model|cors|csrf|security|config|settings|" + r"(^|/)(main|app|deps|dependencies)\.py$|(^|/)(server|app|index)\.[cm]?[jt]sx?$)" +) + + +def _git(*args): + cmd = [GIT, *args] + # Feste git-Aufrufe ohne Shell, Argumente nur aus diesem Skript bzw. Git-Referenzen. + # RUF100 im noqa: Apps ohne S-Regeln würden das noqa sonst als unnötig melden. + r = subprocess.run(cmd, capture_output=True, text=True, check=False) # noqa: S603, RUF100 # nosec B603 + return r.returncode, r.stdout.strip() + + +def _rx(name, default): + return re.compile(os.environ.get(name) or default) + + +def find_base(arg): + """Gibt den Merge-Base-Commit zurück oder None.""" + candidates = [arg] if arg else [] + if not arg: + rc, up = _git("rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") + if rc == 0 and up: + candidates.append(up) + candidates += ["origin/HEAD", "origin/main", "origin/master", "main", "master"] + for ref in candidates: + rc, _ = _git("rev-parse", "--verify", "-q", ref + "^{commit}") + if rc != 0: + continue + rc, mb = _git("merge-base", ref, "HEAD") + if rc == 0 and mb: + return mb + return None + + +def changed_files(base): + files = set() + for args in ( + ("diff", "--name-only", "--no-renames", base, "HEAD"), + ("diff", "--name-only", "--no-renames", "HEAD"), + ("ls-files", "--others", "--exclude-standard"), + ): + rc, out = _git(*args) + if rc == 0 and out: + files.update(line for line in out.splitlines() if line) + return sorted(files) + + +def classify(files): + tooling = _rx("TOOLING_PATHS", DEFAULT_TOOLING) + deps = _rx("DEPS_PATHS", DEFAULT_DEPS) + docs = _rx("DOCS_PATHS", DEFAULT_DOCS) + sec = _rx("SECURITY_PATHS", DEFAULT_SECURITY) + groups = {"tooling": [], "deps": [], "docs": [], "code": [], "security": []} + for f in files: + if tooling.search(f): + groups["tooling"].append(f) + elif deps.search(f): + groups["deps"].append(f) + elif docs.search(f): + groups["docs"].append(f) + else: + groups["code"].append(f) + if sec.search(f): + groups["security"].append(f) + return groups + + +def targets(base, groups): + mutations = os.environ.get("MUTATIONS") or "security-mutations.toml" + has_mut = os.path.isfile(mutations) + if base is None or groups["tooling"]: + first = "secrets" if base is None else "secrets-diff" + t = [first, "lint", "test", "sast", "audit"] + return t + (["mutation"] if has_mut else []) + t = ["secrets-diff"] + if groups["code"] or groups["deps"]: + t += ["lint", "test"] + if groups["code"]: + t.append("sast-diff") + if groups["deps"]: + t.append("audit") + if groups["security"] and has_mut: + t.append("mutation") + return t + + +def explain(base, files, groups, t): + if base is None: + return "prepush: keine Basis (Upstream/main) gefunden -> volles Programm" + short = _git("rev-parse", "--short", base)[1] + lines = [f"prepush: Basis {short}, {len(files)} geänderte Datei(en)"] + labels = { + "tooling": "Baseline-Werkzeug (-> volles check)", + "deps": "Abhängigkeiten (-> audit)", + "security": "sicherheitsrelevant (-> mutation)", + "code": "Code (-> lint, test, sast-diff)", + "docs": "ohne Code (-> nur Secrets)", + } + for key, label in labels.items(): + if groups[key]: + more = " ..." if len(groups[key]) > 4 else "" + sample = ", ".join(groups[key][:4]) + more + lines.append(f" {label}: {sample}") + lines.append(" Ziele: " + " ".join(t)) + return "\n".join(lines) + + +def main(argv): + if len(argv) < 2 or argv[1] not in {"targets", "files", "range", "explain"}: + print(__doc__, file=sys.stderr) + return 2 + mode, arg = argv[1], (argv[2] if len(argv) > 2 else "") + base = find_base(arg) + if mode == "range": + print(f"{base}..HEAD" if base else "") + return 0 + files = changed_files(base) if base else [] + if mode == "files": + print("\n".join(f for f in files if os.path.isfile(f))) + return 0 + groups = classify(files) + t = targets(base, groups) + print(" ".join(t) if mode == "targets" else explain(base, files, groups, t)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv))