Migration to kubernetes + vault-agent

.gitignore

@@ -1,2 +1,8 @@
+.idea/*
 *.swp
 *.log
+varnish_list
+helm/charts/*
+Chart.lock
+app/*.log
+app/http-broadcaster

Dockerfile

@@ -0,0 +1,61 @@
+########################
+# BASE
+########################
+FROM golang:1.23.3-alpine as base
+
+ENV CGO_ENABLED=0 \
+    GOOS=linux \
+    GOARCH=amd64
+
+ARG APP_UID=1000
+ARG APP_GID=1000
+RUN addgroup -S app -g ${APP_GID} && adduser -u ${APP_UID} -S -D -G app app
+
+RUN apk update \
+    && apk add --no-cache bash ca-certificates tzdata curl \
+    && update-ca-certificates
+
+ENV TZ="Europe/Paris"
+
+COPY ./app /app
+WORKDIR /app
+COPY ./docker/config/env.local /vault/secrets/.env
+
+RUN go mod download && go mod verify
+
+########################
+# BUILD
+########################
+FROM base as build-env
+
+RUN go build -ldflags="-w -s" -o /http-broadcaster
+
+########################
+# PROD ENV ###
+########################
+FROM alpine:3.20 as prod
+
+ARG APP_UID=1000
+ARG APP_GID=1000
+RUN addgroup -S app -g ${APP_GID} && adduser -u ${APP_UID} -S -D -G app app
+
+RUN apk update \
+    && apk add --no-cache bash ca-certificates tzdata curl \
+    && update-ca-certificates
+
+ENV TZ="Europe/Paris"
+COPY --from=build-env /http-broadcaster /usr/local/bin/http-broadcaster
+RUN chmod +x /usr/local/bin/http-broadcaster
+RUN mkdir /app && chown ${APP_UID}:${APP_GID} /app
+
+USER app
+
+########################
+# DEV
+########################
+FROM base as dev
+
+COPY --from=build-env /http-broadcaster /usr/local/bin/http-broadcaster
+RUN chmod +x /usr/local/bin/http-broadcaster
+
+ENTRYPOINT ["http-broadcaster"]

README.md

@@ -3,26 +3,33 @@
 ## Description
 Un démon simple écrit en Go qui prend une requête PURGE en entrée et la transmet à plusieurs serveurs varnish.
 
-## Installation
+## Déploiement
-L'installation se fait via un playbook dans le dépot [ansible-dev](https://gitlab.infolegale.net/infrastructure/ansible-dev/-/blob/master/playbooks/http-broadcaster.yml).
+Le projet se déploie via les pipelines Gitlab en stg et en prod, via un déclenchement manuel.
-Le rôle va se charger de déposer l'artefact créé par la pipeline de ce projet, le fichier de service systemd et démarrer le démon.
+Le sidecar vault va déposer un fichier (/vault/secrets/.env) contenant les variables d'environnements.
-Le rôle dépose également la liste des serveurs varnish à côté du binaire, au format :
+
-```
+## Configuration
-http://10.13.32.1:6081,http://10.13.32.2:6081
+Arguments de lancement :
-```
+* -l (--log) : emplacement du fichier de log. (Default : /app/http-broadcaster.log)
+* -e (--envfile) : emplacement du fichier de variables d'environnement. (Default : /vault/secrets/.env)
+* --metrics : active l'exposition des métriques prometheus sur /metrics. (Default : false)
+
+### Variables d'environement
+
+La liste de serveurs Varnish peut être fournie directement dans un fichier d'env :
+* VARNISH_SERVERS: list of varnish backend servers. Ex "http://10.13.32.1:6081,http://10.13.32.2:6081"
+* CLIENT_LIST : list of IPs in CIDR format (0.0.0.0/32) of authorized client to do purge/ban requests.
 
 ## Fonctionnalites
 
-* Génère la liste des serveurs Varnish en lisant le fichier "varnish" présent à côté du binaire.
+* Génère la liste des serveurs Varnish à partir des variables d'environnement.
 * Ecoute sur le port 6081.
 * Healthcheck disponible sur l'uri /healthcheck pour vérifier son bon fonctionnement. Renvoie un code HTTP 200 et le message "OK".
-* Traite toutes les requêtes entrantes comme l'url à purger dans varnish. Par exemple un appel sur http://10.13.101.11:6081/codes/api/greffes/0101 entrainera une purge de l'uri "/codes/api/greffes/0101" sur les serveurs Varnish.
+* Metriques Prometheus disponible (désactivée par défaut).
+* Traite les requêtes entrantes en récupérant 3 éléments et en les intégrant à la requête transmise aux serveurs Varnish :
+- La méthode (PURGE ou BAN par exemple)
+- L'url : / pour BAN, /codes/api/greffes/0101 par exemple pour PURGE.
+- Le header X-Cache-Tags : dans le cas d'un BAN ce header contient une valeur.
 
 ## Usage
 Les interactions se font via le protocol HTTP. Les applications où les utilisateurs envoient une requête de méthode PURGE vers le démon.
 Une fois le traitement d'une requête effectuée, le démon renvoie 200 si tout est ok, 405 dans le cas contraire.
-
-## Roadmap
-* Aller chercher la liste des varnish dans vault.
-* Ajouter une forme d'authentification.
-* Ajouter d'autres possibilités que l'envoi à Varnish.

app/.env.example

@@ -0,0 +1,2 @@
+CLIENT_LIST="127.0.0.1/32"
+VARNISH_SERVERS="127.0.0.1:6081"

app/Http/utils.go

@@ -0,0 +1,114 @@
+// Package http provides functions to handle incoming HTTP requests
+package http
+
+import (
+	prometheus "http-broadcaster/Prometheus"
+	tools "http-broadcaster/Tools"
+	varnish "http-broadcaster/Varnish"
+	"io"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// logRequest print the requests and wanted informations in log file
+func logRequest(t time.Time, r *http.Request, s int, h map[string]string) {
+	// Test if X-Cache-Tags header is empty
+	if len(h) == 0 {
+		log.Printf("%s %s - - %s \"%s %s %s\" %d 0 \"-\" \"%s\" %d\n",
+			r.Host,
+			r.Header["X-Forwarded-For"][0],
+			t.Format("[02/Jan/2006:15:04:05 -0700]"),
+			r.Method,
+			r.URL.Path,
+			r.Proto,
+			s,
+			r.UserAgent(),
+			time.Since(t).Milliseconds(),
+		)
+	} else {
+		var header string
+		if h["X-Cache-Tags"] != "" {
+			header = h["X-Cache-Tags"]
+		} else {
+			header = h["ApiPlatform-Ban-Regex"]
+		}
+		log.Printf("%s %s - - %s \"%s %s %s\" %d 0 \"-\" \"%s\" %d %s\n",
+			r.Host,
+			r.Header["X-Forwarded-For"][0],
+			t.Format("[02/Jan/2006:15:04:05 -0700]"),
+			r.Method,
+			r.URL.Path,
+			r.Proto,
+			s,
+			r.UserAgent(),
+			time.Since(t).Milliseconds(),
+			header,
+		)
+	}
+}
+
+// checkAllowedIP verify if the IPs is authorized to do BAN/PURGE request.
+func checkAllowedIP(ip string) bool {
+	return tools.IPAllowed(ip)
+}
+
+// RequestHandler handles requests to broadcast to all varnish instances.
+func RequestHandler(w http.ResponseWriter, r *http.Request) {
+	var tag = make(map[string]string)
+	ipAddress := r.RemoteAddr
+	// check x-forwarded-for instead of RemoteAddr header because kube
+	//ip, err := netip.ParseAddr(r.Header["X-Forwarded-For"][0])
+	fwdAddress := r.Header.Get("X-Forwarded-For")
+	if fwdAddress != "" {
+		// Case there is a single IP in the header
+		ipAddress = fwdAddress
+
+		ips := strings.Split(fwdAddress, ",")
+		if len(ips) > 1 {
+			ipAddress = ips[0]
+		}
+	}
+
+	// If IP is not authorized to do purge/ban requests, respond with 401.
+	if !checkAllowedIP(ipAddress) {
+		log.Printf("Client ip not authorized : %v", ipAddress)
+		w.WriteHeader(401)
+		_, _ = io.WriteString(w, strconv.Itoa(401))
+		return
+	}
+	// If metrics are not enabled, return 404 on /metrics path.
+	if r.URL.Path == "/metrics" && !prometheus.MetricsEnabled {
+		w.WriteHeader(404)
+		_, _ = io.WriteString(w, strconv.Itoa(404))
+		return
+	}
+	t := time.Now()
+	url := r.URL.String()
+	method := r.Method
+	h := r.Header.Get("X-Cache-Tags")
+	if h != "" {
+		tag["X-Cache-Tags"] = h
+	}
+	h = r.Header.Get("ApiPlatform-Ban-Regex")
+	if h != "" {
+		tag["ApiPlatform-Ban-Regex"] = h
+	}
+	status := varnish.SendToVarnish(method, url, tag)
+	if prometheus.MetricsEnabled {
+		prometheus.IncrementClientCounterVec(method)
+	}
+	// Return HTTP code 405 if not all varnish servers returned 200.
+	if status != 200 {
+		w.WriteHeader(405)
+	}
+	logRequest(t, r, status, tag)
+	_, _ = io.WriteString(w, strconv.Itoa(status))
+}
+
+// HealthHandler handles healthcheck requests and return 200.
+func HealthHandler(w http.ResponseWriter, _ *http.Request) {
+	_, _ = io.WriteString(w, "OK")
+}

app/Prometheus/utils.go

@@ -0,0 +1,76 @@
+// Package prometheus provides useful functions to initialize and populate metrics
+package prometheus
+
+import (
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// Metrics structure for storing counter
+type Metrics struct {
+	ClientHTTPReqs  *prometheus.CounterVec
+	BackendHTTPReqs *prometheus.CounterVec
+}
+
+var (
+	// HTTPCounter Metrics
+	HTTPCounter = initializeHTTPReqCounter()
+	// Reg is a custom registry to have more control on metrics exported.
+	Reg = prometheus.NewRegistry()
+	// MetricsEnabled is the flag used to enable the prometheus metrics backend.
+	MetricsEnabled = false
+)
+
+// InitMetrics enable the metrics functionality if the flags is passed as an argument
+func InitMetrics(m bool) {
+	if m {
+		MetricsEnabled = true
+		initPrometheusRegistry()
+		// Define custom promhttp handler that expose just our custom registry.
+		http.Handle("/metrics", promhttp.HandlerFor(Reg, promhttp.HandlerOpts{
+			EnableOpenMetrics: true,
+			Registry:          Reg,
+		}))
+	}
+}
+
+// InitPrometheusRegistry initialize registry and counters if metrics flag pass as argument.
+func initPrometheusRegistry() {
+	// We use a custom registry to better now what metrics are exposed.
+	Reg = prometheus.NewRegistry()
+	Reg.MustRegister(HTTPCounter.ClientHTTPReqs)
+	Reg.MustRegister(HTTPCounter.BackendHTTPReqs)
+	Reg.MustRegister(collectors.NewBuildInfoCollector())
+}
+
+// IncrementClientCounterVec increments the counter with method label provided.
+func IncrementClientCounterVec(m string) {
+	HTTPCounter.ClientHTTPReqs.WithLabelValues(m).Inc()
+}
+
+// IncrementBackendCounterVec increments the counter with method label provided.
+func IncrementBackendCounterVec(m string) {
+	HTTPCounter.BackendHTTPReqs.WithLabelValues(m).Inc()
+}
+
+// InitializeHTTPReqCounter inits the httpReqs counter that will be exported.
+func initializeHTTPReqCounter() *Metrics {
+	HTTPCounters := &Metrics{
+		ClientHTTPReqs: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "client_http_requests_total",
+			Help: "How many HTTP requests processed, partitioned by HTTP method.",
+		},
+			[]string{"method"},
+		),
+		BackendHTTPReqs: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "backend_http_requests_total",
+			Help: "How many HTTP requests sent to backend, partitioned by HTTP method.",
+		},
+			[]string{"method"},
+		),
+	}
+	return HTTPCounters
+}

app/Tools/utils.go

@@ -0,0 +1,64 @@
+package Tools
+
+import (
+	"log"
+	"net/netip"
+	"os"
+	"strings"
+
+	"github.com/joho/godotenv"
+)
+
+var (
+	// ClientList contains IPs/networks authorized to do purge/ban
+	ClientList []netip.Prefix
+)
+
+// ReadDotEnvFile reads environment variables from .env file
+func ReadDotEnvFile(f string) {
+	err := godotenv.Load(f)
+	if err != nil {
+		log.Fatal("Error loading .env file")
+	}
+}
+
+// InitLog ensure log file exists and set appropriate flags (remove timestamp at start of line).
+func InitLog(p string) {
+	logFile, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		panic(err)
+	}
+	log.SetOutput(logFile)
+	log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime))
+}
+
+// InitAllowedIPList initialize the list of client authorized to do purge/ban requests
+func InitAllowedIPList(l string) []netip.Prefix {
+	list := []netip.Prefix{}
+	if l != "" {
+		sliceData := strings.Split(l, ",")
+		for i := 0; i < len(sliceData); i++ {
+			t, err := netip.ParsePrefix(sliceData[i])
+			if err != nil {
+				panic(err)
+			}
+			list = append(list, t)
+		}
+		return list
+	}
+	return list
+}
+
+// IPAllowed check if the IP is authorized to do BAN/PURGE requests
+func IPAllowed(ip string) bool {
+	ipAddr, err := netip.ParseAddr(ip)
+	if err != nil {
+		log.Printf("Ip address wrong format %s", err)
+	}
+	for i := 0; i < len(ClientList); i++ {
+		if ClientList[i].Contains(ipAddr) {
+			return true
+		}
+	}
+	return false
+}

app/Varnish/utils.go

@@ -0,0 +1,64 @@
+// Package varnish provides functions to build the list of varnish servers that will be used
+package varnish
+
+import (
+	prometheus2 "http-broadcaster/Prometheus"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+var (
+	// VarnishList contains the list of varnish servers.
+	VarnishList []string
+	status      = 200
+)
+
+// InitializeVarnishList sets varnishList variable according to the LIST_METHOD env var
+func InitializeVarnishList(l string) []string {
+	data := os.Getenv("VARNISH_SERVERS")
+	sliceData := strings.Split(string(data), ",")
+	return sliceData
+}
+
+// SendToVarnish send to all varnish servers define in varnishList the request with the PURGE or BAN method
+// and the X-Cache-Tags header if necessary.
+func SendToVarnish(method string, url string, tag map[string]string) int {
+	status = 200
+
+	// Take url to ban as argument.
+	// Loop over the list of Varnish servers and send PURGE request to each.
+	// Update status variable to check if servers have successfully purge url.
+	for i := 0; i < len(VarnishList); i++ {
+		client := &http.Client{
+			Timeout: 10 * time.Second,
+		}
+		// sanitize varnish server host.
+		domain := strings.Trim(VarnishList[i], "\r\n")
+		req, err := http.NewRequest(method, domain+url, nil)
+		if err != nil {
+			log.Println("Create new request : ", err)
+		}
+		// If X-Cache-Tags header is not empty with pass it to varnish.
+		if tag["X-Cache-Tags"] != "" {
+			req.Header.Add("X-Cache-Tags", tag["X-Cache-Tags"])
+		}
+		if tag["ApiPlatform-Ban-Regex"] != "" {
+			req.Header.Add("ApiPlatform-Ban-Regex", tag["ApiPlatform-Ban-Regex"])
+		}
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Println("Send new request : ", err)
+		}
+		if prometheus2.MetricsEnabled {
+			prometheus2.IncrementBackendCounterVec(method)
+		}
+		if resp.StatusCode != 200 {
+			status = 405
+		}
+		resp.Body.Close()
+	}
+	return status
+}

app/Vault/configuration.go

@@ -1,61 +0,0 @@
-package Vault
-
-import (
-    "fmt"
-    "os"
-    vault "github.com/hashicorp/vault/api"
-    auth "github.com/hashicorp/vault/api/auth/approle"
-)
-
-// getVarnishList retrieve the list of varnish servers to send PURGE to.
-// It uses the AppRole authentication method.
-func getVarnishList() (string, error) {
-	config := vault.DefaultConfig() // modify for more granular configuration
-
-	client, err := vault.NewClient(config)
-	if err != nil {
-		return "", fmt.Errorf("unable to initialize Vault client: %w", err)
-	}
-
-        // Get roleID and secretID from ENV vars
-	roleID := os.Getenv("APPROLE_ROLE_ID")
-	if roleID == "" {
-		return "", fmt.Errorf("no role ID was provided in APPROLE_ROLE_ID env var")
-	}
-	secretID := os.Getenv("APPROLE_SECRET_ID")
-	if secretID == "" {
-		return "", fmt.Errorf("no secret ID was provided in APPROLE_SECRET_ID env var")
-	}
-
-	appRoleAuth, err := auth.NewAppRoleAuth(
-		roleID,
-		secretID,
-		auth.WithWrappingToken(), // Only required if the secret ID is response-wrapped.
-	)
-	if err != nil {
-		return "", fmt.Errorf("unable to initialize AppRole auth method: %w", err)
-	}
-
-	authInfo, err := client.Auth().Login(context.Background(), appRoleAuth)
-	if err != nil {
-		return "", fmt.Errorf("unable to login to AppRole auth method: %w", err)
-	}
-	if authInfo == nil {
-		return "", fmt.Errorf("no auth info was returned after login")
-	}
-
-	// get secret from the default mount path for KV v2 in dev mode, "secret"
-	secret, err := client.KVv2("app").Get(context.Background(), "http-broadcaster/stg/varnish_list")
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %w", err)
-	}
-
-	// data map can contain more than one key-value pair,
-	// in this case we're just grabbing one of them
-	value, ok := secret.Data["list"].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data["list"], secret.Data["list"])
-	}
-
-	return value, nil
-}

app/go.mod

@@ -1,3 +1,25 @@
 module http-broadcaster
 
-go 1.13
+go 1.20
+
+require (
+	github.com/alexflint/go-arg v1.5.1
+	github.com/joho/godotenv v1.5.1
+	github.com/prometheus/client_golang v1.20.5
+)
+
+require (
+	github.com/alexflint/go-scalar v1.2.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+)

app/go.sum

@@ -0,0 +1,138 @@
+github.com/alecthomas/kingpin/v2 v2.3.1 h1:ANLJcKmQm4nIaog7xdr/id6FM6zm5hHnfZrvtKPxqGg=
+github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc=
+github.com/alexflint/go-arg v1.4.3 h1:9rwwEBpMXfKQKceuZfYcwuc/7YY7tWJbFsgG5cAU/uo=
+github.com/alexflint/go-arg v1.4.3/go.mod h1:3PZ/wp/8HuqRZMUUgu7I+e1qcpUbvmS258mRXkFH4IA=
+github.com/alexflint/go-arg v1.5.0 h1:rwMKGiaQuRbXfZNyRUvIfke63QvOBt1/QTshlGQHohM=
+github.com/alexflint/go-arg v1.5.0/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8=
+github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y=
+github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8=
+github.com/alexflint/go-scalar v1.1.0 h1:aaAouLLzI9TChcPXotr6gUhq+Scr8rl0P9P4PnltbhM=
+github.com/alexflint/go-scalar v1.1.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
+github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
+github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
+github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
+github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
+github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
+github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
+github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
+github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
+github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI=
+github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg=
+github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
+github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI=
+github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
+github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
+github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
+github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
+github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
+github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
+github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
+github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/xhit/go-str2duration v1.2.0 h1:BcV5u025cITWxEQKGWr1URRzrcXtu7uk8+luz3Yuhwc=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

app/main.go

@@ -2,81 +2,34 @@
 package main
 
 import (
-	"io"
+	h "http-broadcaster/Http"
+	prometheus2 "http-broadcaster/Prometheus"
+	tools "http-broadcaster/Tools"
+	varnish "http-broadcaster/Varnish"
 	"log"
 	"net/http"
 	"os"
-	"strings"
-)
 
-const (
+	"github.com/alexflint/go-arg"
-	// MethodPurge declaration for Varnish.
-	MethodPurge = "PURGE"
 )
 
 var (
-	varnishList = MakeVarnishList()
+	args struct {
-	status      = "200 Purged"
+		Log     string `arg:"-l,--logfile" help:"location of output logfile." default:"/app/http-broadcaster.log"`
+		EnvFile string `arg:"-e,--envfile" help:"location of file containing environment variables." default:"/vault/secrets/.env"`
+		Metrics bool   `arg:"--metrics" help:"enable prometheus exporter on /metrics." default:"false"`
+	}
 )
 
-// MakeVarnishList reads the list of varnish servers from a file on disk.
-func MakeVarnishList() []string {
-	Data, err := os.ReadFile("./varnish")
-	if err != nil {
-		log.Fatal(err)
-	}
-	sliceData := strings.Split(string(Data), ",")
-	return sliceData
-}
-
-// PurgeHandler handles PURGE request to broadcast it to all varnish instances.
-func PurgeHandler(w http.ResponseWriter, r *http.Request) {
-	url := r.URL.String()
-	remoteAddr := r.RemoteAddr
-	status := SendToVarnish(url)
-	if status != "200 Purged" {
-		w.WriteHeader(405)
-	}
-	log.Println(remoteAddr + " Requested purge on " + url + " : " + status)
-	io.WriteString(w, status)
-}
-
-// SendToVarnish send to all varnish servers define in varnishList the PURGE request.
-func SendToVarnish(url string) string {
-	status = "200 Purged"
-	// Take url to ban as argument.
-	// Loop over the list of Varnish servers and send PURGE request to each.
-	// Update status variable to check if servers have successfully purge url.
-	for i := 0; i < len(varnishList); i++ {
-		client := &http.Client{}
-		domain := strings.Trim(varnishList[i], "\r\n")
-		req, err := http.NewRequest(MethodPurge, domain+url, nil)
-		if err != nil {
-			log.Fatal("Create new request : %s", err)
-		}
-		resp, err := client.Do(req)
-		if err != nil {
-			log.Fatal("Send new request : %s", err)
-		}
-		if resp.StatusCode != 200 {
-			status = "405 Not Allowed"
-		}
-	}
-	return status
-}
-
-// HealthHandler handles healthcheck requests and return 200.
-func HealthHandler(w http.ResponseWriter, _ *http.Request) {
-	io.WriteString(w, "OK")
-}
-
 func main() {
-	logFile, err := os.OpenFile("./log/purge.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	arg.MustParse(&args)
-	if err != nil {
+	tools.InitLog(args.Log)
-		panic(err)
+	tools.ReadDotEnvFile(args.EnvFile)
-	}
+	tools.ClientList = tools.InitAllowedIPList(os.Getenv("CLIENT_LIST"))
-	log.SetOutput(logFile)
+	varnish.VarnishList = varnish.InitializeVarnishList(os.Getenv("VARNISH_SERVERS"))
-	http.HandleFunc("/", PurgeHandler)
+	prometheus2.InitMetrics(args.Metrics)
-	http.HandleFunc("/healthcheck", HealthHandler)
+
+	http.HandleFunc("/", h.RequestHandler)
+	http.HandleFunc("/healthcheck", h.HealthHandler)
 	log.Fatal(http.ListenAndServe(":6081", nil))
 }

app/varnish

@@ -1 +0,0 @@
-http://10.13.32.1:6081,http://10.13.32.2:6081

docker-compose.yml

@@ -0,0 +1,18 @@
+---
+
+version: '0.1'
+services:
+    broadcaster:
+        container_name: httpbroadcaster
+        build:
+            context: ./
+            target: dev
+        ports:
+            - "6081:6081"
+        volumes:
+            - ./app/:/app/
+        networks:
+            - http-broadcaster
+networks:
+    http-broadcaster:
+        driver: bridge

docker/config/env.local

@@ -0,0 +1 @@
+VARNISH_SERVERS="http://192.168.1.2:6081,http://192.168.1.1:6081"