Revision 3eee19a17a23489bb9461e0526f42d7f71da35ed authored by Quentin Monnet on 16 January 2023, 13:56:21 UTC, committed by Quentin Monnet on 17 January 2023, 09:41:25 UTC
Once upon a time, Cilium docs used the openapi Sphinx add-on to generate its API reference based on the code. And things were good. One day, Dependabot raised a security alert, stating that Mistune v2.0.2 was vulnerable to catastrophic backtracking [0] - this is a regex parsing thing. Mistune was a dependency to m2r, an add-on to parse Markdown in Sphinx, which in turn was a dependency to openapi. The easy path would have been to update m2r to use the latest, fixed Mistune version; but m2r was incompatible with Mistune >= 2.0.0, and also it was no longer in development. There was a fork, m2r2, which had little activity, and would avoid the security issue by very simply pinning the Mistune version to 0.8.4 (which would either fail to build Cilium's reference correctly, or bring some incompatibilities with other dependencies, at this point the narrator does not remember for sure). There was a fork of the fork, sphinx-mdinclude. We could use that project to update openapi, except that it was not compatible with recent versions of docutils, and that this would cause openapi's test suite to fail to pass. ... So we ended up forking the openapi repository to update the dependency to sphinx-mdinclude locally, and this is what we've been using since last summer. And things were good again. But things are even better when they go upstream [citation needed]. We also filed the issue for docutils compatibility in sphinx-mdinclude [1]. It was fixed (thanks!). We submitted a PR to have openapi switch to sphinx-mdinclude [2]. It was adjusted (thanks!), merged, and a new tag was created. Now at last, we can switch back to the upstream version of openapi! [And the build system lived happily ever after.] [0]: https://github.com/advisories/GHSA-fw3v-x4f2-v673 [1]: https://github.com/omnilib/sphinx-mdinclude/issues/8 [2]: https://github.com/sphinx-contrib/openapi/pull/127 I did _not_ run `make -C Documentation update-requirements`, because the resulting changes seemed to break the Netlify preview [3]. I stuck to openapi and bumped sphinx-mdinclude to >= 0.5.2, as required by openapi. [3] https://app.netlify.com/sites/docs-cilium-io/deploys/63c55fcc5531c6000838b87c Signed-off-by: Quentin Monnet <quentin@isovalent.com>
1 parent 4ec82bb
Makefile.defs
# Copyright Authors of Cilium
# SPDX-License-Identifier: Apache-2.0
SHELL := /bin/bash
.SHELLFLAGS := -eu -o pipefail -c
# define a function replacing spaces with commas in a list
empty :=
space := $(empty) $(empty)
comma := ,
join-with-comma = $(subst $(space),$(comma),$(strip $1))
ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
RELATIVE_DIR := $(shell echo $(realpath .) | sed "s;$(ROOT_DIR)[/]*;;")
include $(ROOT_DIR)/Makefile.quiet
PREFIX?=/usr
BINDIR?=$(PREFIX)/bin
CNIBINDIR?=/opt/cni/bin
CNICONFDIR?=/etc/cni/net.d
LIBDIR?=$(PREFIX)/lib
LOCALSTATEDIR?=/var
RUNDIR?=/var/run
CONFDIR?=/etc
NATIVE_ARCH = $(shell GOARCH= $(GO) env GOARCH)
export GOARCH ?= $(NATIVE_ARCH)
INSTALL = install
CONTAINER_ENGINE?=docker
DOCKER_FLAGS?=
DOCKER_BUILD_FLAGS?=
# use gsed if avaiable, otherwise use sed.
# gsed is needed for MacOS to make in-place replacement work correctly.
SED ?= $(if $(shell command -v gsed), gsed, sed)
# Set DOCKER_DEV_ACCOUNT with "cilium" by default
ifeq ($(DOCKER_DEV_ACCOUNT),)
DOCKER_DEV_ACCOUNT=cilium
endif
ifneq ($(CI_BUILD),)
DOCKER_IMAGE_SUFFIX=-ci
DOCKER_IMAGE_TAG=$(shell git rev-parse HEAD)
endif
# Set DOCKER_IMAGE_TAG with "latest" by default
ifeq ($(DOCKER_IMAGE_TAG),)
DOCKER_IMAGE_TAG=latest
endif
ifeq ($(shell uname -m),aarch64)
ETCD_IMAGE=quay.io/coreos/etcd:v3.3.20-arm64
else
ETCD_IMAGE=quay.io/coreos/etcd:v3.3.20
endif
CONSUL_IMAGE=consul:1.7.2
GO ?= go
# go build/test/clean flags
# these are declared here so they are treated explicitly
# as non-immediate variables
GO_BUILD_FLAGS =
GO_TEST_FLAGS =
GO_CLEAN_FLAGS =
GO_BUILD_LDFLAGS =
# go build/test -tags values
GO_TAGS_FLAGS = osusergo
# This is declared here as it is needed to change the covermode depending on if
# RACE is specified.
GOTEST_COVER_OPTS =
# By default, just print go test output immediately to the terminal. If tparse
# is installed, use it to format the output. This is either silent silent (V=0)
# or gives some signal about ongoing progress by teeing to the terminal (V=1).
GOTEST_FORMATTER ?= cat
ifneq ($(shell command -v tparse),)
GOTEST_COVER_OPTS += -json
GOTEST_FORMATTER = tparse
ifneq ($(V),0)
GOTEST_FORMATTER += -follow
endif
endif
GOLANGCILINT_WANT_VERSION = 1.48.0
GOLANGCILINT_IMAGE_SHA = sha256:5e4256fd5001c1c6ec3de3d31fdae2fa2af6dd21ebbb86b3ddfe1925376af326
GOLANGCILINT_VERSION = $(shell golangci-lint version 2>/dev/null)
VERSION = $(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION)
VERSION_MAJOR = $(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION | cut -d. -f1)
# Use git only if in a Git repo
ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/.git),)
GIT_VERSION = $(shell git show -s --format='format:%h %aI')
else
GIT_VERSION = $(shell cat $(ROOT_DIR)/GIT_VERSION)
endif
FULL_BUILD_VERSION = $(VERSION) $(GIT_VERSION)
GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/version.ciliumVersion=$(FULL_BUILD_VERSION)"
ifeq ($(NOSTRIP),)
# Note: these options will not remove annotations needed for stack
# traces, so panic backtraces will still be readable.
#
# -w: Omit the DWARF symbol table.
# -s: Omit the symbol table and debug information.
GO_BUILD_LDFLAGS += -s -w
endif
ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/images/cilium/Dockerfile),)
CILIUM_ENVOY_REF=$(shell sed -E -e 's/^FROM (--[^ ]* )*([^ ]*) as cilium-envoy/\2/p;d' < $(ROOT_DIR)/images/cilium/Dockerfile)
CILIUM_ENVOY_SHA=$(shell echo $(CILIUM_ENVOY_REF) | sed -E -e 's/[^/]*\/[^:]*:([^:@-]*).*/\1/p;d')
GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/envoy.RequiredEnvoyVersionSHA=$(CILIUM_ENVOY_SHA)"
endif
# Use git only if in a Git repo, otherwise find the files from the file system
BPF_SRCFILES_IGNORE = bpf/.gitignore
ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/.git),)
BPF_SRCFILES := $(shell git ls-files $(ROOT_DIR)/bpf/ | LC_ALL=C sort | tr "\n" ' ')
else
# this line has to be in-sync with bpf/.gitignore, please note usage of make patterns like `%.i`
BPF_SRCFILES_IGNORE += bpf/%.i bpf/%.s bpf/.rebuild_all
BPF_SRCFILES := $(shell find $(ROOT_DIR)/bpf/ -type f | LC_ALL=C sort | tr "\n" ' ')
endif
# ROOT_DIR can be either `../` or absolute path, each of these need to be stripped
BPF_SRCFILES := $(filter-out $(BPF_SRCFILES_IGNORE),$(subst ../,,$(subst $(ROOT_DIR)/,,$(BPF_SRCFILES))))
CILIUM_DATAPATH_SHA256=$(shell cd $(ROOT_DIR); cat $(BPF_SRCFILES) | sha256sum | awk '{print $$1}')
GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/datapath/loader.DatapathSHA256=$(CILIUM_DATAPATH_SHA256)"
# Set -mod=vendor if running >= go 1.13 or if GO111MODULE is set.
# A go build is being executed with go modules if:
# * The go command is invoked with GO111MODULE=on environment variable set.
# * The go command is invoked in a directory outside of the $GOPATH/src tree
# and the environment variable GO111MODULE unset (or explicitly set to 'auto').
ifeq ($(GO111MODULE),on)
GO_BUILD_FLAGS += -mod=vendor
GO_TEST_FLAGS += -mod=vendor
GO_CLEAN_FLAGS += -mod=vendor
else
GO_MAJOR_VERSION_GE_1 := $(shell expr `$(GO) version | grep -E 'go[0-9]+' -o | sed 's/go//g'` \>= 1)
ifeq ($(GO_MAJOR_VERSION_GE_1),1)
GO_MINOR_VERSION_GE_13 := $(shell expr `$(GO) version | grep -E 'go[^ ]+' -o | sed 's/go1.//g'` \>= 13)
ifeq ($(GO_MINOR_VERSION_GE_13),1)
GO_BUILD_FLAGS += -mod=vendor
GO_TEST_FLAGS += -mod=vendor
GO_CLEAN_FLAGS += -mod=vendor
endif
endif
endif
GO_BUILD = CGO_ENABLED=0 $(GO) build
# Support CGO cross-compiling for amd64 and arm64 targets
CGO_CC =
CROSS_ARCH =
ifneq ($(GOARCH),$(NATIVE_ARCH))
CROSS_ARCH = $(GOARCH)
endif
ifeq ($(CROSS_ARCH),arm64)
CGO_CC = CC=aarch64-linux-gnu-gcc
else ifeq ($(CROSS_ARCH),amd64)
CGO_CC = CC=x86_64-linux-gnu-gcc
endif
GO_BUILD_WITH_CGO = CGO_ENABLED=1 $(CGO_CC) $(GO) build
ifneq ($(RACE),)
GO_BUILD_FLAGS += -race
GO_TEST_FLAGS += -race
GOTEST_COVER_OPTS += -covermode=atomic
# GO_BUILD becomes GO_BUILD_WITH_CGO as `-race` requires CGO
GO_BUILD = $(GO_BUILD_WITH_CGO)
ifeq ($(LOCKDEBUG),)
LOCKDEBUG=1
endif
else
GOTEST_COVER_OPTS += -covermode=count
endif
ifneq ($(LOCKDEBUG),)
GO_TAGS_FLAGS += lockdebug
endif
GO_BUILD_FLAGS += -ldflags '$(GO_BUILD_LDFLAGS) $(EXTRA_GO_BUILD_LDFLAGS)' -tags=$(call join-with-comma,$(GO_TAGS_FLAGS)) $(EXTRA_GO_BUILD_FLAGS)
GO_TEST_FLAGS += -tags=$(call join-with-comma,$(GO_TAGS_FLAGS))
ifeq ($(NOOPT),1)
GO_BUILD_FLAGS += -gcflags="all=-N -l"
endif
GO_BUILD += $(GO_BUILD_FLAGS)
GO_BUILD_WITH_CGO += $(GO_BUILD_FLAGS)
GO_TEST = CGO_ENABLED=0 $(GO) test $(GO_TEST_FLAGS)
GO_CLEAN = $(GO) clean $(GO_CLEAN_FLAGS)
GO_VET = $(GO) vet
GO_LIST = $(GO) list
ifeq ($(BASE_IMAGE),)
BASE_IMAGE=scratch
endif
HELM_TOOLBOX_VERSION ?= "v1.0.1"
HELM_TOOLBOX_SHA ?= "a1483fc036b15c290326ba7ed36b8f0b714e185913b72a6074c04225d74e034c"
HELM_TOOLBOX_IMAGE ?= "quay.io/cilium/helm-toolbox:$(HELM_TOOLBOX_VERSION)@sha256:$(HELM_TOOLBOX_SHA)"
define print_help_line
@printf " \033[36m%-29s\033[0m %s.\n" $(1) $(2)
endef
define print_help_from_makefile
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9][a-zA-Z0-9 _-]*:.*?##/ { split($$1, targets, " "); for (i in targets) { printf " \033[36m%-28s\033[0m %s\n", targets[i], $$2 } } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
endef
![swh spinner](/static/img/swh-spinner.gif)
Computing file changes ...