Skip to content
Hands-on IT IT Drills

Day 7: Finalizers and External Resources

Goal: The site-operator registers every StaticSite with an external DNS registrar and reliably removes the entry again via a finalizer – even if the service goes down in between · Duration: approx. 120 min · Prerequisite: Day 6

Briefing

  • Owner references only clean up Kubernetes objects. Everything outside the cluster (DNS, cloud load balancers, Vault policies) needs a finalizer.
  • A finalizer is just a string in metadata.finalizers. As long as it is there, the API server only sets deletionTimestamp on delete – the object stays.
  • The controller sees deletionTimestamp, cleans up externally and removes its finalizer. Only then does the object disappear.
  • If the external service goes down, the finalizer stays: retry with backoff instead of orphaned data.
kubectl delete  ->  deletionTimestamp set, finalizers: [web.lab.example/finalizer]
                        |
        Reconcile: DELETE /records/<name>.<ns>  --(503)-->  error, backoff, retry
                        | 204/404
              RemoveFinalizer + Update  ->  API server deletes the object for good

Drill 1: The external service – a lab registrar

New day, new directory: you keep working on a copy of Day 6.

source ~/op-lab/env.sh
cp -r ~/op-lab/tag06-site-operator ~/op-lab/tag07-site-operator
mkdir -p ~/op-lab/registrar && cd ~/op-lab/registrar
cat > go.mod <<'EOF'
module example.com/registrar

go 1.26
EOF

The registrar is a small HTTP service with a map as storage. It plays the DNS provider you would manage via API at a customer – including a maintenance window via /admin/outage.

cat > main.go <<'EOF'
// registrar is a lab "DNS registrar": an external service that the
// site-operator (Day 7) manages via HTTP. Data lives in memory only.
//
//  PUT    /records/<name>  {"target":"..."}  -> 201 new, 200 changed
//  GET    /records                           -> 200 all entries
//  DELETE /records/<name>                    -> 204 or 404
//  PUT    /admin/outage   / DELETE /admin/outage -> outage on/off (503)
package main

import (
    "encoding/json"
    "flag"
    "log/slog"
    "net/http"
    "os"
    "regexp"
    "sort"
    "sync"
)

// Record is a DNS entry: name -> target (e.g. service URL).
type Record struct {
    Name   string `json:"name"`
    Target string `json:"target"`
}

type registrar struct {
    mu      sync.Mutex
    records map[string]Record
    outage  bool
}

var validName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$`)

func main() {
    addr := flag.String("addr", "127.0.0.1:18090", "Listen-Adresse")
    flag.Parse()
    log := slog.New(slog.NewTextHandler(os.Stderr, nil))
    r := &registrar{records: map[string]Record{}}

    mux := http.NewServeMux()
    mux.HandleFunc("GET /records", r.list)
    mux.HandleFunc("PUT /records/{name}", r.put)
    mux.HandleFunc("DELETE /records/{name}", r.del)
    mux.HandleFunc("PUT /admin/outage", r.setOutage(true))
    mux.HandleFunc("DELETE /admin/outage", r.setOutage(false))

    log.Info("registrar startet", "addr", *addr)
    h := logging(log, r.guard(mux))
    if err := http.ListenAndServe(*addr, h); err != nil {
        log.Error("beendet", "err", err)
        os.Exit(1)
    }
}

// guard simulates an outage: everything except /admin/ then returns 503.
func (r *registrar) guard(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
        r.mu.Lock()
        down := r.outage
        r.mu.Unlock()
        if down && req.URL.Path != "/admin/outage" {
            http.Error(w, "registrar: Wartung", http.StatusServiceUnavailable)
            return
        }
        next.ServeHTTP(w, req)
    })
}

func (r *registrar) list(w http.ResponseWriter, _ *http.Request) {
    r.mu.Lock()
    out := make([]Record, 0, len(r.records))
    for _, rec := range r.records {
        out = append(out, rec)
    }
    r.mu.Unlock()
    sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
    writeJSON(w, http.StatusOK, out)
}

func (r *registrar) put(w http.ResponseWriter, req *http.Request) {
    name := req.PathValue("name")
    var body struct {
        Target string `json:"target"`
    }
    err := json.NewDecoder(http.MaxBytesReader(w, req.Body, 4096)).Decode(&body)
    if err != nil || body.Target == "" || !validName.MatchString(name) {
        http.Error(w, "ungültiger Name oder target fehlt", http.StatusBadRequest)
        return
    }
    r.mu.Lock()
    _, existed := r.records[name]
    rec := Record{Name: name, Target: body.Target}
    r.records[name] = rec
    r.mu.Unlock()
    status := http.StatusCreated
    if existed {
        status = http.StatusOK // PUT ist idempotent
    }
    writeJSON(w, status, rec)
}

func (r *registrar) del(w http.ResponseWriter, req *http.Request) {
    name := req.PathValue("name")
    r.mu.Lock()
    _, ok := r.records[name]
    delete(r.records, name)
    r.mu.Unlock()
    if !ok {
        http.Error(w, "nicht gefunden", http.StatusNotFound)
        return
    }
    w.WriteHeader(http.StatusNoContent)
}

func (r *registrar) setOutage(on bool) http.HandlerFunc {
    return func(w http.ResponseWriter, _ *http.Request) {
        r.mu.Lock()
        r.outage = on
        r.mu.Unlock()
        w.WriteHeader(http.StatusNoContent)
    }
}

func writeJSON(w http.ResponseWriter, status int, v any) {
    w.Header().Set("Content-Type", "application/json")
    w.WriteHeader(status)
    _ = json.NewEncoder(w).Encode(v)
}

func logging(log *slog.Logger, next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
        log.Info("request", "method", req.Method, "path", req.URL.Path)
        next.ServeHTTP(w, req)
    })
}
EOF

Build it and start it in the background – it keeps running all day:

go vet ./... && go build -o ~/op-lab/bin/registrar .
registrar > ~/op-lab/registrar/registrar.log 2>&1 & REG_PID=$!
curl -s -X PUT localhost:18090/records/demo.default \
  -d '{"target":"http://demo.default.svc"}' -w ' %{http_code}\n'
curl -s localhost:18090/records
curl -s -X DELETE localhost:18090/records/demo.default -w '%{http_code}\n'
curl -s -X DELETE localhost:18090/records/demo.default -w ' %{http_code}\n'

Expected:

{"name":"demo.default","target":"http://demo.default.svc"}
 201
[{"name":"demo.default","target":"http://demo.default.svc"}]
204
nicht gefunden
 404

Why? The second DELETE returns 404 (nicht gefunden, “not found”) – for your operator that means “already gone” and therefore success. Cleanup must be idempotent, because Reconcile can be repeated at any time.

Save your progress? Sign up for free

Drill 2: A client with a timeout

The operator does not speak net/http directly in Reconcile, but through a small package. That keeps the controller readable and the service replaceable.

cd ~/op-lab/tag07-site-operator
mkdir -p internal/registrar
cat > internal/registrar/client.go <<'EOF'
// Package registrar talks to the lab DNS registrar (code/registrar).
package registrar

import (
    "bytes"
    "context"
    "encoding/json"
    "fmt"
    "net/http"
    "time"
)

// Client manages DNS entries via HTTP. An empty BaseURL turns it off.
type Client struct {
    BaseURL string
    HTTP    *http.Client
}

// New returns a client with a timeout – a hanging service must not
// block the Reconcile worker.
func New(baseURL string) *Client {
    return &Client{BaseURL: baseURL, HTTP: &http.Client{Timeout: 5 * time.Second}}
}

// Upsert creates or updates the entry (PUT is idempotent).
func (c *Client) Upsert(ctx context.Context, name, target string) error {
    body, err := json.Marshal(map[string]string{"target": target})
    if err != nil {
        return err
    }
    return c.do(ctx, http.MethodPut, name, body, http.StatusOK, http.StatusCreated)
}

// Delete removes the entry. 404 counts as success: the goal is reached.
func (c *Client) Delete(ctx context.Context, name string) error {
    return c.do(ctx, http.MethodDelete, name, nil,
        http.StatusNoContent, http.StatusNotFound)
}

func (c *Client) do(ctx context.Context, method, name string, body []byte,
    ok ...int) error {
    url := fmt.Sprintf("%s/records/%s", c.BaseURL, name)
    req, err := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body))
    if err != nil {
        return err
    }
    resp, err := c.HTTP.Do(req)
    if err != nil {
        return fmt.Errorf("registrar %s %s: %w", method, name, err)
    }
    defer resp.Body.Close()
    for _, code := range ok {
        if resp.StatusCode == code {
            return nil
        }
    }
    return fmt.Errorf("registrar %s %s: HTTP %d", method, name, resp.StatusCode)
}
EOF

Why? Without Timeout, a hanging service can block a Reconcile worker forever – and with MaxConcurrentReconciles=1 (the default) the whole controller then stalls.

The client gets tested along the way – through the finalizer test in envtest (Drill 4).

Save your progress? Sign up for free

Drill 3: Setting and handling the finalizer

All the finalizer logic goes into its own file. If no registrar is configured, the operator does not set a finalizer either – so tests and clusters without a registrar keep running as before.

cd ~/op-lab/tag07-site-operator
cat > internal/controller/finalizer.go <<'EOF'
package controller

import (
    "context"
    "fmt"

    ctrl "sigs.k8s.io/controller-runtime"
    "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    logf "sigs.k8s.io/controller-runtime/pkg/log"

    webv1alpha1 "example.com/site-operator/api/v1alpha1"
)

// siteFinalizer holds on to the StaticSite until the DNS entry is gone.
const siteFinalizer = "web.lab.example/finalizer"

// recordName is the key in the registrar: <name>.<namespace>.
func recordName(site *webv1alpha1.StaticSite) string {
    return site.Name + "." + site.Namespace
}

// registerDNS registers the site with the registrar (PUT is idempotent).
func (r *StaticSiteReconciler) registerDNS(ctx context.Context,
    site *webv1alpha1.StaticSite) error {
    if r.Registrar == nil {
        return nil // --registrar-url leer: DNS-Pflege aus
    }
    url := fmt.Sprintf("http://%s.%s.svc", site.Name, site.Namespace)
    return r.Registrar.Upsert(ctx, recordName(site), url)
}

// reconcileDelete cleans up externally and then releases the object.
func (r *StaticSiteReconciler) reconcileDelete(ctx context.Context,
    site *webv1alpha1.StaticSite) (ctrl.Result, error) {
    if !controllerutil.ContainsFinalizer(site, siteFinalizer) {
        return ctrl.Result{}, nil // nicht (mehr) unsere Aufgabe
    }
    if r.Registrar != nil {
        if err := r.Registrar.Delete(ctx, recordName(site)); err != nil {
            return ctrl.Result{}, err // Finalizer bleibt, Retry mit Backoff
        }
        logf.FromContext(ctx).Info("DNS-Eintrag entfernt", "record", recordName(site))
    }
    controllerutil.RemoveFinalizer(site, siteFinalizer)
    return ctrl.Result{}, r.Update(ctx, site)
}
EOF

The reconciler gets a field for the client (internal/controller/staticsite_controller.go, add the import "example.com/site-operator/internal/registrar"):

type StaticSiteReconciler struct {
    client.Client
    Scheme    *runtime.Scheme
    Recorder  events.EventRecorder
    Registrar *registrar.Client // nil: keine DNS-Pflege
}

In Reconcile, right after the Get, two branches, and after the Kubernetes objects the DNS entry:

    if !site.DeletionTimestamp.IsZero() { // kubectl delete: aufräumen
        return r.reconcileDelete(ctx, &site)
    }
    if r.Registrar != nil && controllerutil.AddFinalizer(&site, siteFinalizer) {
        if err := r.Update(ctx, &site); err != nil {
            return ctrl.Result{}, err
        }
    }

    dep, recErr := r.reconcileResources(ctx, &site)
    if recErr == nil {
        recErr = r.registerDNS(ctx, &site)
    }
    // Status immer schreiben – ... (wie an Tag 5)

Why the finalizer first, then the external resource? The other way around, the operator could crash after the DNS entry exists but before the finalizer is in place – a later delete would leave an orphan behind. AddFinalizer returns true only if it actually changed something; so the Update runs once per object.

Errors from registerDNS end up in recErr – updateStatus from Day 5 turns them into Degraded=True with reason ReconcileError, and the workqueue retries with backoff. Kubebuilder already generated the RBAC rule staticsites/finalizers: update on Day 3; from today on it is needed.

The flag in cmd/main.go is still missing. Add the import "example.com/site-operator/internal/registrar", then next to the other flags:

    var registrarURL string
    flag.StringVar(&registrarURL, "registrar-url", "",
        "Basis-URL des DNS-Registrars, z. B. http://127.0.0.1:18090 (leer: aus)")

and where the reconciler is created:

    var reg *registrar.Client
    if registrarURL != "" {
        reg = registrar.New(registrarURL)
    }
    if err := (&controller.StaticSiteReconciler{
        Client:    mgr.GetClient(),
        Scheme:    mgr.GetScheme(),
        Recorder:  mgr.GetEventRecorder("site-operator"),
        Registrar: reg,
    }).SetupWithManager(mgr); err != nil {
go build ./... && go vet ./... && echo ok

Expected:

ok

Save your progress? Sign up for free

Drill 4: A registrar in the test

envtest knows no registrar. So the suite starts an httptest server that behaves like the real one – and fails at the push of a button.

cat > internal/controller/fakeregistrar_test.go <<'EOF'
package controller

import (
    "net/http"
    "net/http/httptest"
    "sync"
    "sync/atomic"
)

// fakeRegistrar plays the registrar inside the test process: a map instead of DNS.
type fakeRegistrar struct {
    *httptest.Server
    mu      sync.Mutex
    records map[string]bool
    down    atomic.Bool // true: jede Anfrage bekommt 503
}

func newFakeRegistrar() *fakeRegistrar {
    f := &fakeRegistrar{records: map[string]bool{}}
    f.Server = httptest.NewServer(http.HandlerFunc(
        func(w http.ResponseWriter, r *http.Request) {
            if f.down.Load() {
                http.Error(w, "Wartung", http.StatusServiceUnavailable)
                return
            }
            name := r.URL.Path[len("/records/"):]
            f.mu.Lock()
            defer f.mu.Unlock()
            switch r.Method {
            case http.MethodPut:
                f.records[name] = true
                w.WriteHeader(http.StatusOK)
            case http.MethodDelete:
                delete(f.records, name)
                w.WriteHeader(http.StatusNoContent)
            }
        }))
    return f
}

// has reports whether the entry exists.
func (f *fakeRegistrar) has(name string) bool {
    f.mu.Lock()
    defer f.mu.Unlock()
    return f.records[name]
}
EOF

In internal/controller/suite_test.go, add a variable fakeReg *fakeRegistrar to the var block and the import "example.com/site-operator/internal/registrar", then start the reconciler with the registrar and close the server in AfterSuite:

    fakeReg = newFakeRegistrar()
    Expect((&StaticSiteReconciler{
        Client:    mgr.GetClient(),
        Scheme:    mgr.GetScheme(),
        Recorder:  mgr.GetEventRecorder("site-operator"),
        Registrar: registrar.New(fakeReg.URL),
    }).SetupWithManager(mgr)).To(Succeed())
    // ...
var _ = AfterSuite(func() {
    By("Manager und envtest stoppen")
    cancel()
    fakeReg.Close()
    Expect(testEnv.Stop()).To(Succeed())
})

Three specs: happy path, outage on delete, outage on registration. You know the helpers newNamespace and newStaticSite from Day 6.

cat > internal/controller/finalizer_test.go <<'EOF'
package controller

import (
    "time"

    . "github.com/onsi/ginkgo/v2"
    . "github.com/onsi/gomega"
    apierrors "k8s.io/apimachinery/pkg/api/errors"
    "k8s.io/apimachinery/pkg/api/meta"
    "sigs.k8s.io/controller-runtime/pkg/client"

    webv1alpha1 "example.com/site-operator/api/v1alpha1"
)

var _ = Describe("Finalizer und Registrar", func() {
    var (
        site   *webv1alpha1.StaticSite
        record string
    )

    BeforeEach(func(ctx SpecContext) {
        site = newStaticSite(newNamespace(ctx), "dns")
        record = "dns." + site.Namespace
        Expect(k8sClient.Create(ctx, site)).To(Succeed())
        Eventually(func() bool { return fakeReg.has(record) }).Should(BeTrue())
    })

    // gone is satisfied when the StaticSite is gone for good.
    gone := func(ctx SpecContext) func() bool {
        return func() bool {
            err := k8sClient.Get(ctx, client.ObjectKeyFromObject(site), site)
            return apierrors.IsNotFound(err)
        }
    }

    It("setzt den Finalizer und räumt beim Löschen ab", func(ctx SpecContext) {
        Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(site), site)).
            To(Succeed())
        Expect(site.Finalizers).To(ContainElement("web.lab.example/finalizer"))

        Expect(k8sClient.Delete(ctx, site)).To(Succeed())
        Eventually(gone(ctx)).Should(BeTrue())
        Expect(fakeReg.has(record)).To(BeFalse())
    })

    It("hält die Site fest, solange der Registrar ausfällt", func(ctx SpecContext) {
        fakeReg.down.Store(true)
        DeferCleanup(func() { fakeReg.down.Store(false) })

        Expect(k8sClient.Delete(ctx, site)).To(Succeed())
        Consistently(gone(ctx), 2*time.Second).Should(BeFalse())
        Expect(site.DeletionTimestamp).NotTo(BeNil()) // Terminating

        fakeReg.down.Store(false)
        Eventually(gone(ctx), 30*time.Second).Should(BeTrue())
        Expect(fakeReg.has(record)).To(BeFalse())
    })

    It("meldet Degraded, wenn der Eintrag nicht klappt", func(ctx SpecContext) {
        fakeReg.down.Store(true)
        DeferCleanup(func() { fakeReg.down.Store(false) })

        patch := client.MergeFrom(site.DeepCopy()) // Patch: kein Konflikt
        site.Spec.Replicas = 2                     // neue Generation -> Reconcile
        Expect(k8sClient.Patch(ctx, site, patch)).To(Succeed())
        Eventually(func() string {
            _ = k8sClient.Get(ctx, client.ObjectKeyFromObject(site), site)
            c := meta.FindStatusCondition(site.Status.Conditions,
                webv1alpha1.ConditionDegraded)
            if c == nil || c.Status != "True" {
                return ""
            }
            return c.Message
        }).Should(ContainSubstring("HTTP 503"))
    })
})
EOF
go test ./internal/controller/ -count=1 -v -ginkgo.no-color -ginkgo.focus=Finalizer \
  2>&1 | grep -E '^(Ran|SUCCESS|FAIL|ok)'

Expected:

Ran 3 of 17 Specs in 8.822 seconds
SUCCESS! -- 3 Passed | 0 Failed | 0 Pending | 14 Skipped
ok      example.com/site-operator/internal/controller   9.237s

Why does the second test hold up? envtest has no garbage collector, but finalizers are evaluated by the API server itself: as long as the list is not empty, the object stays with a deletionTimestamp. You are testing this for real here, not simulating it.

Save your progress? Sign up for free

Drill 5: Live – operator, registrar, API server

Terminal 1: envtest-up --crds config/crd/bases (from ~/op-lab/tag07-site-operator). Terminal 3: the operator, this time with the registrar:

source ~/op-lab/env.sh && cd ~/op-lab/tag07-site-operator
go run ./cmd/main.go -health-probe-bind-address=0 \
  --registrar-url=http://127.0.0.1:18090 2>&1 | tee run.log

Terminal 2:

kubectl apply -f lab/demo.yaml
kubectl get staticsite demo -o jsonpath='{.metadata.finalizers}{"\n"}'
curl -s localhost:18090/records; echo
kubectl delete staticsite demo
curl -s localhost:18090/records; echo
grep -E 'PUT|DELETE' ~/op-lab/registrar/registrar.log | tail -2

Expected (shortened):

staticsite.web.lab.example/demo created
["web.lab.example/finalizer"]
[{"name":"demo.default","target":"http://demo.default.svc"}]
staticsite.web.lab.example "demo" deleted from default namespace
[]
... msg=request method=PUT path=/records/demo.default
... msg=request method=DELETE path=/records/demo.default

kubectl delete waits until the object is really gone – that is, until your operator has removed the finalizer. run.log shows DNS-Eintrag entfernt ... "record": "demo.default" (“DNS entry removed”) for this.

Save your progress? Sign up for free

Drill 6: Outage and a stuck finalizer

First the outage: registrar in maintenance, then delete.

kubectl apply -f lab/demo.yaml && sleep 2
curl -s -X PUT localhost:18090/admin/outage
kubectl delete staticsite demo --wait=false
sleep 4
kubectl get staticsite demo \
  -o jsonpath='{.metadata.deletionTimestamp} {.metadata.finalizers}{"\n"}'
grep 'Reconciler error' run.log | tail -1 | grep -o '"error": "[^"]*"'

Expected:

staticsite.web.lab.example/demo created
staticsite.web.lab.example "demo" deleted from default namespace
2026-10-10T08:48:44Z ["web.lab.example/finalizer"]
"error": "registrar DELETE demo.default: HTTP 503"

The site is “Terminating”, the entry stays in place, and the operator keeps retrying at growing intervals. End the maintenance – seconds later it is gone:

curl -s -X DELETE localhost:18090/admin/outage
sleep 3; kubectl get staticsite demo

Expected:

Error from server (NotFound): staticsites.web.lab.example "demo" not found

Now the classic from operations: the operator is gone (uninstalled, CrashLoop, broken RBAC), the finalizer stays. Create the site, then stop the operator in Terminal 3 with Ctrl+C:

kubectl apply -f lab/demo.yaml && sleep 2
# Terminal 3: Ctrl+C
kubectl delete staticsite demo --timeout=5s
kubectl get site

Expected:

staticsite.web.lab.example "demo" deleted from default namespace
error: timed out waiting for the condition on staticsites/demo
NAME   READY   AVAILABLE   URL                       AGE
demo           False       http://demo.default.svc   9s

The right fix: start the operator again – it cleans up. The emergency fix, if it never comes back: remove the finalizer by hand. Then you have to clean up the external resource:

kubectl patch staticsite demo --type=json \
  -p '[{"op":"remove","path":"/metadata/finalizers"}]'
kubectl get site
curl -s localhost:18090/records; echo
curl -s -X DELETE localhost:18090/records/demo.default -w '%{http_code}\n'

Expected:

staticsite.web.lab.example/demo patched
No resources found in default namespace.
[{"name":"demo.default","target":"http://demo.default.svc"}]
204

Warning: Removing a finalizer by hand is an intervention with consequences: the external resource is orphaned. At a customer, this belongs in the runbook – with the step “check and clean up externally”.

Start the operator in Terminal 3 again (same command as in Drill 5) for the checkpoint.

Save your progress? Sign up for free

Checkpoint

cd ~/op-lab/tag07-site-operator
ok() { echo "✔ $1"; }; nok() { echo "✘ $1"; }
go test ./internal/... -count=1 >/dev/null 2>&1 && ok "Tests grün" || nok "Tests"
curl -sf localhost:18090/records >/dev/null && ok "Registrar läuft" || nok "Registrar"
kubectl apply -f lab/demo.yaml >/dev/null; sleep 2
kubectl get site demo -o jsonpath='{.metadata.finalizers}' | grep -q finalizer \
  && ok "Finalizer gesetzt" || nok "Finalizer fehlt (läuft der Operator?)"
curl -s localhost:18090/records | grep -q '"demo.default"' \
  && ok "DNS-Eintrag angelegt" || nok "DNS-Eintrag fehlt"
kubectl delete site demo --timeout=10s >/dev/null && ok "Site gelöscht" || nok "hängt"
curl -s localhost:18090/records | grep -q '"demo.default"' \
  && nok "DNS-Eintrag noch da" || ok "DNS-Eintrag entfernt"

Expected:

✔ Tests grün
✔ Registrar läuft
✔ Finalizer gesetzt
✔ DNS-Eintrag angelegt
✔ Site gelöscht
✔ DNS-Eintrag entfernt

Clean up: stop the operator (Terminal 3) and envtest-up (Terminal 1) with Ctrl+C. You do not need the registrar on Day 8: kill $REG_PID.

Challenge

Build a documented emergency exit: if a StaticSite carries the annotation web.lab.example/skip-cleanup: "true", the operator removes its finalizer on delete without asking the registrar – and logs that. Test it with the registrar down.

Hint: The branch belongs in reconcileDelete before the Delete call; an annotation can still be set even on an object with a deletionTimestamp.

Solution

In finalizer.go, a constant and the branch:

// skipCleanupAnnotation: emergency exit if the registrar is gone for good.
const skipCleanupAnnotation = "web.lab.example/skip-cleanup"

    if site.Annotations[skipCleanupAnnotation] == "true" {
        logf.FromContext(ctx).Info("Aufräumen übersprungen", "record", recordName(site))
    } else if r.Registrar != nil {
        if err := r.Registrar.Delete(ctx, recordName(site)); err != nil {

Restart the operator in Terminal 3 (Drill 5), then with the registrar down:

kubectl apply -f lab/demo.yaml && sleep 2
curl -s -X PUT localhost:18090/admin/outage
kubectl annotate staticsite demo web.lab.example/skip-cleanup=true
kubectl delete staticsite demo --timeout=10s
grep -o 'Aufräumen übersprungen' run.log
curl -s -X DELETE localhost:18090/admin/outage
curl -s localhost:18090/records; echo

Expected:

staticsite.web.lab.example/demo created
staticsite.web.lab.example/demo annotated
staticsite.web.lab.example "demo" deleted from default namespace
Aufräumen übersprungen
[{"name":"demo.default","target":"http://demo.default.svc"}]

The entry deliberately stays in place – that is exactly what the annotation says. Whoever sets it cleans up externally themselves.

Save your progress? Sign up for free

Enterprise Note

  • Finalizer names are API: once rolled out, you have to keep handling them in every later version, otherwise old objects get stuck.
  • External calls need a timeout, idempotency (PUT/404 = ok) and an unambiguous mapping (e.g. a tag with the StaticSite’s UID) – otherwise a new operator deletes someone else’s entries.
  • During long outages, exponential backoff grows up to ~16 min; for known maintenance windows, prefer RequeueAfter with a fixed interval and a “deletions stuck” metric.
  • Namespace deletion blocks as long as an object in it has a finalizer – never uninstall an operator before its CRs (Helm hooks, ordering in GitOps).
  • Credentials for the real DNS/cloud API belong in a Secret (or workload identity), never in flags or ConfigMaps.