Revision 3eee19a17a23489bb9461e0526f42d7f71da35ed authored by Quentin Monnet on 16 January 2023, 13:56:21 UTC, committed by Quentin Monnet on 17 January 2023, 09:41:25 UTC
Once upon a time, Cilium docs used the openapi Sphinx add-on to generate
its API reference based on the code. And things were good.

One day, Dependabot raised a security alert, stating that Mistune v2.0.2
was vulnerable to catastrophic backtracking [0] - this is a regex
parsing thing. Mistune was a dependency to m2r, an add-on to parse
Markdown in Sphinx, which in turn was a dependency to openapi.

The easy path would have been to update m2r to use the latest, fixed
Mistune version; but m2r was incompatible with Mistune >= 2.0.0, and
also it was no longer in development.

There was a fork, m2r2, which had little activity, and would avoid the
security issue by very simply pinning the Mistune version to 0.8.4
(which would either fail to build Cilium's reference correctly, or bring
some incompatibilities with other dependencies, at this point the
narrator does not remember for sure).

There was a fork of the fork, sphinx-mdinclude. We could use that
project to update openapi, except that it was not compatible with recent
versions of docutils, and that this would cause openapi's test suite to
fail to pass.

... So we ended up forking the openapi repository to update the
dependency to sphinx-mdinclude locally, and this is what we've been
using since last summer. And things were good again.

But things are even better when they go upstream [citation needed]. We
also filed the issue for docutils compatibility in sphinx-mdinclude [1].
It was fixed (thanks!). We submitted a PR to have openapi switch to
sphinx-mdinclude [2]. It was adjusted (thanks!), merged, and a new tag
was created.

Now at last, we can switch back to the upstream version of openapi!
[And the build system lived happily ever after.]

[0]: https://github.com/advisories/GHSA-fw3v-x4f2-v673
[1]: https://github.com/omnilib/sphinx-mdinclude/issues/8
[2]: https://github.com/sphinx-contrib/openapi/pull/127

I did _not_ run `make -C Documentation update-requirements`, because the
resulting changes seemed to break the Netlify preview [3]. I stuck to
openapi and bumped sphinx-mdinclude to >= 0.5.2, as required by openapi.

[3] https://app.netlify.com/sites/docs-cilium-io/deploys/63c55fcc5531c6000838b87c

Signed-off-by: Quentin Monnet <quentin@isovalent.com>
1 parent 4ec82bb
Raw File
Makefile.defs
# Copyright Authors of Cilium
# SPDX-License-Identifier: Apache-2.0

SHELL := /bin/bash
.SHELLFLAGS := -eu -o pipefail -c

# define a function replacing spaces with commas in a list
empty :=
space := $(empty) $(empty)
comma := ,
join-with-comma = $(subst $(space),$(comma),$(strip $1))

ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
RELATIVE_DIR := $(shell echo $(realpath .) | sed "s;$(ROOT_DIR)[/]*;;")
include $(ROOT_DIR)/Makefile.quiet

PREFIX?=/usr
BINDIR?=$(PREFIX)/bin
CNIBINDIR?=/opt/cni/bin
CNICONFDIR?=/etc/cni/net.d
LIBDIR?=$(PREFIX)/lib
LOCALSTATEDIR?=/var
RUNDIR?=/var/run
CONFDIR?=/etc

NATIVE_ARCH = $(shell GOARCH= $(GO) env GOARCH)
export GOARCH ?= $(NATIVE_ARCH)

INSTALL = install

CONTAINER_ENGINE?=docker
DOCKER_FLAGS?=
DOCKER_BUILD_FLAGS?=

# use gsed if avaiable, otherwise use sed.
# gsed is needed for MacOS to make in-place replacement work correctly.
SED ?= $(if $(shell command -v gsed), gsed, sed)

# Set DOCKER_DEV_ACCOUNT with "cilium" by default
ifeq ($(DOCKER_DEV_ACCOUNT),)
    DOCKER_DEV_ACCOUNT=cilium
endif

ifneq ($(CI_BUILD),)
    DOCKER_IMAGE_SUFFIX=-ci
    DOCKER_IMAGE_TAG=$(shell git rev-parse HEAD)
endif

# Set DOCKER_IMAGE_TAG with "latest" by default
ifeq ($(DOCKER_IMAGE_TAG),)
    DOCKER_IMAGE_TAG=latest
endif

ifeq ($(shell uname -m),aarch64)
    ETCD_IMAGE=quay.io/coreos/etcd:v3.3.20-arm64
else
    ETCD_IMAGE=quay.io/coreos/etcd:v3.3.20
endif

CONSUL_IMAGE=consul:1.7.2

GO ?= go

# go build/test/clean flags
# these are declared here so they are treated explicitly
# as non-immediate variables
GO_BUILD_FLAGS =
GO_TEST_FLAGS =
GO_CLEAN_FLAGS =
GO_BUILD_LDFLAGS =
# go build/test -tags values
GO_TAGS_FLAGS = osusergo

# This is declared here as it is needed to change the covermode depending on if
# RACE is specified.
GOTEST_COVER_OPTS =

# By default, just print go test output immediately to the terminal. If tparse
# is installed, use it to format the output. This is either silent silent (V=0)
# or gives some signal about ongoing progress by teeing to the terminal (V=1).
GOTEST_FORMATTER ?= cat
ifneq ($(shell command -v tparse),)
	GOTEST_COVER_OPTS += -json
	GOTEST_FORMATTER = tparse
ifneq ($(V),0)
	GOTEST_FORMATTER += -follow
endif
endif

GOLANGCILINT_WANT_VERSION = 1.48.0
GOLANGCILINT_IMAGE_SHA = sha256:5e4256fd5001c1c6ec3de3d31fdae2fa2af6dd21ebbb86b3ddfe1925376af326
GOLANGCILINT_VERSION = $(shell golangci-lint version 2>/dev/null)

VERSION = $(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION)
VERSION_MAJOR = $(shell cat $(dir $(lastword $(MAKEFILE_LIST)))/VERSION | cut -d. -f1)
# Use git only if in a Git repo
ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/.git),)
    GIT_VERSION = $(shell git show -s --format='format:%h %aI')
else
    GIT_VERSION = $(shell cat $(ROOT_DIR)/GIT_VERSION)
endif
FULL_BUILD_VERSION = $(VERSION) $(GIT_VERSION)
GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/version.ciliumVersion=$(FULL_BUILD_VERSION)"

ifeq ($(NOSTRIP),)
    # Note: these options will not remove annotations needed for stack
    # traces, so panic backtraces will still be readable.
    #
    # -w: Omit the DWARF symbol table.
    # -s: Omit the symbol table and debug information.
    GO_BUILD_LDFLAGS += -s -w
endif

ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/images/cilium/Dockerfile),)
    CILIUM_ENVOY_REF=$(shell sed -E -e 's/^FROM (--[^ ]* )*([^ ]*) as cilium-envoy/\2/p;d' < $(ROOT_DIR)/images/cilium/Dockerfile)
    CILIUM_ENVOY_SHA=$(shell echo $(CILIUM_ENVOY_REF) | sed -E -e 's/[^/]*\/[^:]*:([^:@-]*).*/\1/p;d')
    GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/envoy.RequiredEnvoyVersionSHA=$(CILIUM_ENVOY_SHA)"
endif

# Use git only if in a Git repo, otherwise find the files from the file system
BPF_SRCFILES_IGNORE = bpf/.gitignore
ifneq ($(wildcard $(dir $(lastword $(MAKEFILE_LIST)))/.git),)
    BPF_SRCFILES := $(shell git ls-files $(ROOT_DIR)/bpf/ | LC_ALL=C sort | tr "\n" ' ')
else
    # this line has to be in-sync with bpf/.gitignore, please note usage of make patterns like `%.i`
    BPF_SRCFILES_IGNORE += bpf/%.i bpf/%.s bpf/.rebuild_all
    BPF_SRCFILES := $(shell find $(ROOT_DIR)/bpf/ -type f | LC_ALL=C sort | tr "\n" ' ')
endif

# ROOT_DIR can be either `../` or absolute path, each of these need to be stripped
BPF_SRCFILES := $(filter-out $(BPF_SRCFILES_IGNORE),$(subst ../,,$(subst $(ROOT_DIR)/,,$(BPF_SRCFILES))))
CILIUM_DATAPATH_SHA256=$(shell cd $(ROOT_DIR); cat $(BPF_SRCFILES) | sha256sum | awk '{print $$1}')
GO_BUILD_LDFLAGS += -X "github.com/cilium/cilium/pkg/datapath/loader.DatapathSHA256=$(CILIUM_DATAPATH_SHA256)"

# Set -mod=vendor if running >= go 1.13 or if GO111MODULE is set.
# A go build is being executed with go modules if:
# * The go command is invoked with GO111MODULE=on environment variable set.
# * The go command is invoked in a directory outside of the $GOPATH/src tree
#   and the environment variable GO111MODULE unset (or explicitly set to 'auto').
ifeq ($(GO111MODULE),on)
    GO_BUILD_FLAGS += -mod=vendor
    GO_TEST_FLAGS += -mod=vendor
    GO_CLEAN_FLAGS += -mod=vendor
else
    GO_MAJOR_VERSION_GE_1 := $(shell expr `$(GO) version | grep -E 'go[0-9]+' -o | sed 's/go//g'` \>= 1)
    ifeq ($(GO_MAJOR_VERSION_GE_1),1)
        GO_MINOR_VERSION_GE_13 := $(shell expr `$(GO) version | grep -E 'go[^ ]+' -o | sed 's/go1.//g'` \>= 13)
        ifeq ($(GO_MINOR_VERSION_GE_13),1)
            GO_BUILD_FLAGS += -mod=vendor
            GO_TEST_FLAGS += -mod=vendor
            GO_CLEAN_FLAGS += -mod=vendor
        endif
    endif
endif

GO_BUILD = CGO_ENABLED=0 $(GO) build

# Support CGO cross-compiling for amd64 and arm64 targets
CGO_CC =
CROSS_ARCH =
ifneq ($(GOARCH),$(NATIVE_ARCH))
    CROSS_ARCH = $(GOARCH)
endif
ifeq ($(CROSS_ARCH),arm64)
    CGO_CC = CC=aarch64-linux-gnu-gcc
else ifeq ($(CROSS_ARCH),amd64)
    CGO_CC = CC=x86_64-linux-gnu-gcc
endif
GO_BUILD_WITH_CGO = CGO_ENABLED=1 $(CGO_CC) $(GO) build

ifneq ($(RACE),)
    GO_BUILD_FLAGS += -race
    GO_TEST_FLAGS += -race
    GOTEST_COVER_OPTS += -covermode=atomic

    # GO_BUILD becomes GO_BUILD_WITH_CGO as `-race` requires CGO
    GO_BUILD = $(GO_BUILD_WITH_CGO)
    ifeq ($(LOCKDEBUG),)
        LOCKDEBUG=1
    endif
else
    GOTEST_COVER_OPTS += -covermode=count
endif

ifneq ($(LOCKDEBUG),)
    GO_TAGS_FLAGS += lockdebug
endif

GO_BUILD_FLAGS += -ldflags '$(GO_BUILD_LDFLAGS) $(EXTRA_GO_BUILD_LDFLAGS)' -tags=$(call join-with-comma,$(GO_TAGS_FLAGS)) $(EXTRA_GO_BUILD_FLAGS)
GO_TEST_FLAGS += -tags=$(call join-with-comma,$(GO_TAGS_FLAGS))

ifeq ($(NOOPT),1)
    GO_BUILD_FLAGS += -gcflags="all=-N -l"
endif

GO_BUILD += $(GO_BUILD_FLAGS)
GO_BUILD_WITH_CGO += $(GO_BUILD_FLAGS)

GO_TEST = CGO_ENABLED=0 $(GO) test $(GO_TEST_FLAGS)
GO_CLEAN = $(GO) clean $(GO_CLEAN_FLAGS)

GO_VET = $(GO) vet
GO_LIST = $(GO) list

ifeq ($(BASE_IMAGE),)
    BASE_IMAGE=scratch
endif

HELM_TOOLBOX_VERSION ?= "v1.0.1"
HELM_TOOLBOX_SHA ?= "a1483fc036b15c290326ba7ed36b8f0b714e185913b72a6074c04225d74e034c"
HELM_TOOLBOX_IMAGE ?= "quay.io/cilium/helm-toolbox:$(HELM_TOOLBOX_VERSION)@sha256:$(HELM_TOOLBOX_SHA)"

define print_help_line
  @printf "  \033[36m%-29s\033[0m %s.\n" $(1) $(2)
endef

define print_help_from_makefile
  @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9][a-zA-Z0-9 _-]*:.*?##/ { split($$1, targets, " "); for (i in targets) { printf "  \033[36m%-28s\033[0m %s\n", targets[i], $$2 } } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
endef
back to top